From 8825c9c74a67d1cef28b6d2d18d5fbaf40e36f51 Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Thu, 15 Nov 2012 16:34:45 -0500 Subject: [PATCH] Enhance log msg to report referer and user-agent Enhance internally logged messages to report referer and user-agent. Pass the referering URL and METHOD between internal servers (when known), and set the user-agent to be the server type (obj-server, container-server, proxy-server, obj-updater, obj-replicator, container-updater, direct-client, etc.) with the process PID. In conjunction with the transaction ID, it helps to track down which PID from a given system was responsible for initiating the request and what that server was working on to make this request. This has been helpful in tracking down interactions between object, container and account servers. We also take things a bit further performaing a bit of refactoring to consolidate calls to transfer_headers() now that we have a helper method for constructing them. Finally we performed further changes to avoid header key duplication due to string literal header key values and the various objects representing headers for requests and responses. See below for more details. ==== Header Keys There seems to be a bit of a problem with the case of the various string literals used for header keys and the interchangable way standard Python dictionaries, HeaderKeyDict() and HeaderEnvironProxy() objects are used. If one is not careful, a header object of some sort (one that does not normalize its keys, and that is not necessarily a dictionary) can be constructed containing header keys which differ only by the case of their string literals. E.g.: { 'x-trans-id': '1234', 'X-Trans-Id': '5678' } Such an object, when passed to http_connect() will result in an on-the-wire header where the key values are merged together, comma separated, that looks something like: HTTP_X_TRANS_ID: 1234,5678 For some headers in some contexts, this is behavior is desirable. For example, one can also use a list of tuples which enumerate the multiple values a single header should have. However, in almost all of the contexts used in the code base, this is not desirable. This behavior arises from a combination of factors: 1. Header strings are not constants and different lower-case and title-case header strings values are used interchangably in the code at times It might be worth the effort to make a pass through the code to stop using string literals and use constants instead, but there are plusses and minuses to doing that, so this was not attempted in this effort 2. HeaderEnvironProxy() objects report their keys in ".title()" case, but normalize all other key references to the form expected by the Request class's environ field swob.Request.headers fields are HeaderEnvironProxy() objects. 3. HeaderKeyDict() objects report their keys in ".lower()" case, and normalize all other key references to ".lower()" case swob.Response.headers fields are HeaderKeyDict() objects. Depending on which object is used and how it is used, one can end up with such a mismatch. This commit takes the following steps as a (PROPOSED) solution: 1. Change HeaderKeyDict() to normalize using ".title()" case to match HeaderEnvironProxy() 2. Replace standard python dictionary objects with HeaderKeyDict() objects where possible This gives us an object that normalizes key references to avoid fixing the code to normalize the string literals. 3. Fix up a few places to use title case string literals to match the new defaults Change-Id: Ied56a1df83ffac793ee85e796424d7d20f18f469 Signed-off-by: Peter Portante --- swift/common/direct_client.py | 35 ++-- swift/common/middleware/catch_errors.py | 4 +- swift/common/middleware/tempurl.py | 7 +- swift/common/swob.py | 23 ++- swift/container/server.py | 8 +- swift/container/updater.py | 15 +- swift/obj/replicator.py | 19 +- swift/obj/server.py | 51 +++--- swift/obj/updater.py | 4 +- swift/proxy/controllers/account.py | 21 +-- swift/proxy/controllers/base.py | 56 ++++-- swift/proxy/controllers/container.py | 23 +-- swift/proxy/controllers/obj.py | 18 +- test/unit/common/middleware/test_except.py | 6 +- test/unit/common/middleware/test_tempurl.py | 12 +- test/unit/common/test_direct_client.py | 32 +++- test/unit/common/test_swob.py | 68 ++++++- test/unit/container/test_server.py | 14 +- test/unit/obj/test_server.py | 193 ++++++++++++-------- test/unit/proxy/test_server.py | 124 +++++++------ 20 files changed, 459 insertions(+), 274 deletions(-) diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py index f859fa7149..b4954dfbe3 100644 --- a/swift/common/direct_client.py +++ b/swift/common/direct_client.py @@ -18,6 +18,7 @@ Internal client library for making calls directly to the servers rather than through the proxy. """ +import os import socket from httplib import HTTPException from time import time @@ -30,6 +31,7 @@ from swiftclient import ClientException, json_loads from swift.common.utils import normalize_timestamp from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \ is_success, is_server_error +from swift.common.swob import HeaderKeyDict def quote(value, safe='/'): @@ -38,6 +40,14 @@ def quote(value, safe='/'): return _quote(value, safe) +def gen_headers(hdrs_in=None, add_ts=False): + hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict() + if add_ts: + hdrs_out['X-Timestamp'] = normalize_timestamp(time()) + hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid() + return hdrs_out + + def direct_get_account(node, part, account, marker=None, limit=None, prefix=None, delimiter=None, conn_timeout=5, response_timeout=15): @@ -68,7 +78,8 @@ def direct_get_account(node, part, account, marker=None, limit=None, qs += '&delimiter=%s' % quote(delimiter) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, - 'GET', path, query_string=qs) + 'GET', path, query_string=qs, + headers=gen_headers()) with Timeout(response_timeout): resp = conn.getresponse() if not is_success(resp.status): @@ -107,7 +118,7 @@ def direct_head_container(node, part, account, container, conn_timeout=5, path = '/%s/%s' % (account, container) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, - 'HEAD', path) + 'HEAD', path, headers=gen_headers()) with Timeout(response_timeout): resp = conn.getresponse() resp.read() @@ -157,7 +168,8 @@ def direct_get_container(node, part, account, container, marker=None, qs += '&delimiter=%s' % quote(delimiter) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, - 'GET', path, query_string=qs) + 'GET', path, query_string=qs, + headers=gen_headers()) with Timeout(response_timeout): resp = conn.getresponse() if not is_success(resp.status): @@ -185,10 +197,10 @@ def direct_delete_container(node, part, account, container, conn_timeout=5, headers = {} path = '/%s/%s' % (account, container) - headers['X-Timestamp'] = normalize_timestamp(time()) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, - 'DELETE', path, headers) + 'DELETE', path, + headers=gen_headers(headers, True)) with Timeout(response_timeout): resp = conn.getresponse() resp.read() @@ -220,7 +232,7 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5, path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, - 'HEAD', path) + 'HEAD', path, headers=gen_headers()) with Timeout(response_timeout): resp = conn.getresponse() resp.read() @@ -262,7 +274,7 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5, path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, - 'GET', path, headers=headers) + 'GET', path, headers=gen_headers(headers)) with Timeout(response_timeout): resp = conn.getresponse() if not is_success(resp.status): @@ -328,10 +340,9 @@ def direct_put_object(node, part, account, container, name, contents, headers['Content-Length'] = '0' if isinstance(contents, basestring): contents = [contents] - headers['X-Timestamp'] = normalize_timestamp(time()) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, - 'PUT', path, headers=headers) + 'PUT', path, headers=gen_headers(headers, True)) for chunk in contents: conn.send(chunk) with Timeout(response_timeout): @@ -365,10 +376,9 @@ def direct_post_object(node, part, account, container, name, headers, :raises ClientException: HTTP POST request failed """ path = '/%s/%s/%s' % (account, container, name) - headers['X-Timestamp'] = normalize_timestamp(time()) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, - 'POST', path, headers=headers) + 'POST', path, headers=gen_headers(headers, True)) with Timeout(response_timeout): resp = conn.getresponse() resp.read() @@ -401,10 +411,9 @@ def direct_delete_object(node, part, account, container, obj, headers = {} path = '/%s/%s/%s' % (account, container, obj) - headers['X-Timestamp'] = normalize_timestamp(time()) with Timeout(conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], part, - 'DELETE', path, headers) + 'DELETE', path, headers=gen_headers(headers, True)) with Timeout(response_timeout): resp = conn.getresponse() resp.read() diff --git a/swift/common/middleware/catch_errors.py b/swift/common/middleware/catch_errors.py index 35ff2eb44d..a31975d87a 100644 --- a/swift/common/middleware/catch_errors.py +++ b/swift/common/middleware/catch_errors.py @@ -39,13 +39,13 @@ class CatchErrorsContext(WSGIContext): resp = HTTPServerError(request=Request(env), body='An error occurred', content_type='text/plain') - resp.headers['x-trans-id'] = trans_id + resp.headers['X-Trans-Id'] = trans_id return resp(env, start_response) # make sure the response has the trans_id if self._response_headers is None: self._response_headers = [] - self._response_headers.append(('x-trans-id', trans_id)) + self._response_headers.append(('X-Trans-Id', trans_id)) start_response(self._response_status, self._response_headers, self._response_exc_info) return resp diff --git a/swift/common/middleware/tempurl.py b/swift/common/middleware/tempurl.py index c86f32e01a..bd3fa10a4c 100644 --- a/swift/common/middleware/tempurl.py +++ b/swift/common/middleware/tempurl.py @@ -103,6 +103,7 @@ from urllib import urlencode from urlparse import parse_qs from swift.common.wsgi import make_pre_authed_env +from swift.common.swob import HeaderKeyDict #: Default headers to remove from incoming requests. Simply a whitespace @@ -211,7 +212,7 @@ class TempURL(object): headers = DEFAULT_OUTGOING_REMOVE_HEADERS if 'outgoing_remove_headers' in conf: headers = conf['outgoing_remove_headers'] - headers = [h.lower() for h in headers.split()] + headers = [h.title() for h in headers.split()] #: Headers to remove from outgoing responses. Lowercase, like #: `x-account-meta-temp-url-key`. self.outgoing_remove_headers = [h for h in headers if h[-1] != '*'] @@ -223,7 +224,7 @@ class TempURL(object): headers = DEFAULT_OUTGOING_ALLOW_HEADERS if 'outgoing_allow_headers' in conf: headers = conf['outgoing_allow_headers'] - headers = [h.lower() for h in headers.split()] + headers = [h.title() for h in headers.split()] #: Headers to allow in outgoing responses. Lowercase, like #: `x-matches-remove-prefix-but-okay`. self.outgoing_allow_headers = [h for h in headers if h[-1] != '*'] @@ -474,7 +475,7 @@ class TempURL(object): removed as per the middlware configuration for outgoing responses. """ - headers = dict(headers) + headers = HeaderKeyDict(headers) for h in headers.keys(): remove = h in self.outgoing_remove_headers if not remove: diff --git a/swift/common/swob.py b/swift/common/swob.py index 1099107123..1efc0f1839 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -231,7 +231,7 @@ class HeaderEnvironProxy(UserDict.DictMixin): class HeaderKeyDict(dict): """ - A dict that lower-cases all keys on the way in, so as to be + A dict that title-cases all keys on the way in, so as to be case-insensitive. """ def __init__(self, *args, **kwargs): @@ -242,30 +242,30 @@ class HeaderKeyDict(dict): def update(self, other): if hasattr(other, 'keys'): for key in other.keys(): - self[key.lower()] = other[key] + self[key.title()] = other[key] else: for key, value in other: - self[key.lower()] = value + self[key.title()] = value def __getitem__(self, key): - return dict.get(self, key.lower()) + return dict.get(self, key.title()) def __setitem__(self, key, value): if value is None: - self.pop(key.lower(), None) + self.pop(key.title(), None) elif isinstance(value, unicode): - return dict.__setitem__(self, key.lower(), value.encode('utf-8')) + return dict.__setitem__(self, key.title(), value.encode('utf-8')) else: - return dict.__setitem__(self, key.lower(), str(value)) + return dict.__setitem__(self, key.title(), str(value)) def __contains__(self, key): - return dict.__contains__(self, key.lower()) + return dict.__contains__(self, key.title()) def __delitem__(self, key): - return dict.__delitem__(self, key.lower()) + return dict.__delitem__(self, key.title()) def get(self, key, default=None): - return dict.get(self, key.lower(), default) + return dict.get(self, key.title(), default) def _resp_status_property(): @@ -803,6 +803,9 @@ class Request(object): "Provides the full url of the request" return self.host_url + self.path_qs + def as_referer(self): + return self.method + ' ' + self.url + def path_info_pop(self): """ Takes one path portion (delineated by slashes) from the diff --git a/swift/container/server.py b/swift/container/server.py index 562040d30e..a9ca8ae00e 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -37,7 +37,7 @@ from swift.common.http import HTTP_NOT_FOUND, is_success from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPConflict, \ HTTPCreated, HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \ HTTPPreconditionFailed, HTTPMethodNotAllowed, Request, Response, \ - HTTPInsufficientStorage, HTTPNotAcceptable + HTTPInsufficientStorage, HTTPNotAcceptable, HeaderKeyDict DATADIR = 'containers' @@ -126,12 +126,14 @@ class ContainerController(object): account_ip, account_port = account_host.rsplit(':', 1) new_path = '/' + '/'.join([account, container]) info = broker.get_info() - account_headers = { + account_headers = HeaderKeyDict({ 'x-put-timestamp': info['put_timestamp'], 'x-delete-timestamp': info['delete_timestamp'], 'x-object-count': info['object_count'], 'x-bytes-used': info['bytes_used'], - 'x-trans-id': req.headers.get('x-trans-id', '-')} + 'x-trans-id': req.headers.get('x-trans-id', '-'), + 'user-agent': 'container-server %s' % os.getpid(), + 'referer': req.as_referer()}) if req.headers.get('x-account-override-deleted', 'no').lower() == \ 'yes': account_headers['x-account-override-deleted'] = 'yes' diff --git a/swift/container/updater.py b/swift/container/updater.py index 27d791e4a3..e012b8f766 100644 --- a/swift/container/updater.py +++ b/swift/container/updater.py @@ -61,6 +61,7 @@ class ContainerUpdater(Daemon): self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.rcache = os.path.join(self.recon_cache_path, "container.recon") + self.user_agent = 'container-updater %s' % os.getpid() def get_account_ring(self): """Get the account ring. Load it if it hasn't been yet.""" @@ -269,14 +270,16 @@ class ContainerUpdater(Daemon): """ with ConnectionTimeout(self.conn_timeout): try: + headers = { + 'X-Put-Timestamp': put_timestamp, + 'X-Delete-Timestamp': delete_timestamp, + 'X-Object-Count': count, + 'X-Bytes-Used': bytes, + 'X-Account-Override-Deleted': 'yes', + 'user-agent': self.user_agent} conn = http_connect( node['ip'], node['port'], node['device'], part, - 'PUT', container, - headers={'X-Put-Timestamp': put_timestamp, - 'X-Delete-Timestamp': delete_timestamp, - 'X-Object-Count': count, - 'X-Bytes-Used': bytes, - 'X-Account-Override-Deleted': 'yes'}) + 'PUT', container, headers=headers) except (Exception, Timeout): self.logger.exception(_( 'ERROR account update failed with ' diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index 00fb29a8fe..fd2e77b664 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -271,6 +271,9 @@ class ObjectReplicator(Daemon): self.recon_cache_path = conf.get('recon_cache_path', '/var/cache/swift') self.rcache = os.path.join(self.recon_cache_path, "object.recon") + self.headers = { + 'Content-Length': '0', + 'user-agent': 'obj-replicator %s' % os.getpid()} def _rsync(self, args): """ @@ -389,12 +392,12 @@ class ObjectReplicator(Daemon): success = self.rsync(node, job, suffixes) if success: with Timeout(self.http_timeout): - http_connect( - node['ip'], node['port'], - node['device'], job['partition'], 'REPLICATE', - '/' + '-'.join(suffixes), - headers={'Content-Length': '0'}).\ - getresponse().read() + conn = http_connect(node['ip'], node['port'], + node['device'], + job['partition'], 'REPLICATE', + '/' + '-'.join(suffixes), + headers=self.headers) + conn.getresponse().read() responses.append(success) if not suffixes or (len(responses) == len(job['nodes']) and all(responses)): @@ -435,7 +438,7 @@ class ObjectReplicator(Daemon): resp = http_connect( node['ip'], node['port'], node['device'], job['partition'], 'REPLICATE', - '', headers={'Content-Length': '0'}).getresponse() + '', headers=self.headers).getresponse() if resp.status == HTTP_INSUFFICIENT_STORAGE: self.logger.error(_('%(ip)s/%(device)s responded' ' as unmounted'), node) @@ -469,7 +472,7 @@ class ObjectReplicator(Daemon): node['ip'], node['port'], node['device'], job['partition'], 'REPLICATE', '/' + '-'.join(suffixes), - headers={'Content-Length': '0'}) + headers=self.headers) conn.getresponse().read() self.suffix_sync += len(suffixes) self.logger.update_stats('suffix.syncs', len(suffixes)) diff --git a/swift/obj/server.py b/swift/obj/server.py index 946c8bb484..79ca08fce0 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -46,7 +46,8 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \ HTTPInternalServerError, HTTPNoContent, HTTPNotFound, HTTPNotModified, \ HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \ HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, UTC, \ - HTTPInsufficientStorage, HTTPForbidden, multi_range_iterator + HTTPInsufficientStorage, HTTPForbidden, multi_range_iterator, \ + HeaderKeyDict DATADIR = 'objects' @@ -474,6 +475,7 @@ class ObjectController(object): request :param objdevice: device name that the object is in """ + headers_out['user-agent'] = 'obj-server %s' % os.getpid() full_path = '/%s/%s/%s' % (account, container, obj) if all([host, partition, contdevice]): try: @@ -508,7 +510,7 @@ class ObjectController(object): normalize_timestamp(headers_out['x-timestamp'])), os.path.join(self.devices, objdevice, 'tmp')) - def container_update(self, op, account, container, obj, headers_in, + def container_update(self, op, account, container, obj, request, headers_out, objdevice): """ Update the container when objects are updated. @@ -517,11 +519,12 @@ class ObjectController(object): :param account: account name for the object :param container: container name for the object :param obj: object name - :param headers_in: dictionary of headers from the original request + :param request: the original request object driving the update :param headers_out: dictionary of headers to send in the container request(s) :param objdevice: device name that the object is in """ + headers_in = request.headers conthosts = [h.strip() for h in headers_in.get('X-Container-Host', '').split(',')] contdevices = [d.strip() for d in @@ -543,13 +546,15 @@ class ObjectController(object): else: updates = [] + headers_out['x-trans-id'] = headers_in.get('x-trans-id', '-') + headers_out['referer'] = request.as_referer() for conthost, contdevice in updates: self.async_update(op, account, container, obj, conthost, contpartition, contdevice, headers_out, objdevice) def delete_at_update(self, op, delete_at, account, container, obj, - headers_in, objdevice): + request, objdevice): """ Update the expiring objects container when objects are updated. @@ -557,7 +562,7 @@ class ObjectController(object): :param account: account name for the object :param container: container name for the object :param obj: object name - :param headers_in: dictionary of headers from the original request + :param request: the original request driving the update :param objdevice: device name that the object is in """ # Quick cap that will work from now until Sat Nov 20 17:46:39 2286 @@ -568,8 +573,11 @@ class ObjectController(object): partition = None hosts = contdevices = [None] - headers_out = {'x-timestamp': headers_in['x-timestamp'], - 'x-trans-id': headers_in.get('x-trans-id', '-')} + headers_in = request.headers + headers_out = HeaderKeyDict({ + 'x-timestamp': headers_in['x-timestamp'], + 'x-trans-id': headers_in.get('x-trans-id', '-'), + 'referer': request.as_referer()}) if op != 'DELETE': partition = headers_in.get('X-Delete-At-Partition', None) hosts = headers_in.get('X-Delete-At-Host', '') @@ -626,7 +634,7 @@ class ObjectController(object): return HTTPNotFound(request=request) metadata = {'X-Timestamp': request.headers['x-timestamp']} metadata.update(val for val in request.headers.iteritems() - if val[0].lower().startswith('x-object-meta-')) + if val[0].startswith('X-Object-Meta-')) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() @@ -635,10 +643,10 @@ class ObjectController(object): if old_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, - obj, request.headers, device) + obj, request, device) if old_delete_at: self.delete_at_update('DELETE', old_delete_at, account, - container, obj, request.headers, device) + container, obj, request, device) disk_file.put_metadata(metadata) return HTTPAccepted(request=request) @@ -728,11 +736,11 @@ class ObjectController(object): if new_delete_at: self.delete_at_update( 'PUT', new_delete_at, account, container, obj, - request.headers, device) + request, device) if old_delete_at: self.delete_at_update( 'DELETE', old_delete_at, account, container, obj, - request.headers, device) + request, device) disk_file.put(fd, upload_size, metadata) except DiskFileNoSpace: return HTTPInsufficientStorage(drive=device, request=request) @@ -740,12 +748,12 @@ class ObjectController(object): if not orig_timestamp or \ orig_timestamp < request.headers['x-timestamp']: self.container_update( - 'PUT', account, container, obj, request.headers, - {'x-size': disk_file.metadata['Content-Length'], - 'x-content-type': disk_file.metadata['Content-Type'], - 'x-timestamp': disk_file.metadata['X-Timestamp'], - 'x-etag': disk_file.metadata['ETag'], - 'x-trans-id': request.headers.get('x-trans-id', '-')}, + 'PUT', account, container, obj, request, + HeaderKeyDict({ + 'x-size': disk_file.metadata['Content-Length'], + 'x-content-type': disk_file.metadata['Content-Type'], + 'x-timestamp': disk_file.metadata['X-Timestamp'], + 'x-etag': disk_file.metadata['ETag']}), device) resp = HTTPCreated(request=request, etag=etag) return resp @@ -907,15 +915,14 @@ class ObjectController(object): old_delete_at = int(disk_file.metadata.get('X-Delete-At') or 0) if old_delete_at: self.delete_at_update('DELETE', old_delete_at, account, - container, obj, request.headers, device) + container, obj, request, device) disk_file.put_metadata(metadata, tombstone=True) disk_file.unlinkold(metadata['X-Timestamp']) if not orig_timestamp or \ orig_timestamp < request.headers['x-timestamp']: self.container_update( - 'DELETE', account, container, obj, request.headers, - {'x-timestamp': metadata['X-Timestamp'], - 'x-trans-id': request.headers.get('x-trans-id', '-')}, + 'DELETE', account, container, obj, request, + HeaderKeyDict({'x-timestamp': metadata['X-Timestamp']}), device) resp = response_class(request=request) return resp diff --git a/swift/obj/updater.py b/swift/obj/updater.py index 31185dec10..4fdd02eb97 100644 --- a/swift/obj/updater.py +++ b/swift/obj/updater.py @@ -228,10 +228,12 @@ class ObjectUpdater(Daemon): :param obj: object name being updated :param headers: headers to send with the update """ + headers_out = headers.copy() + headers_out['user-agent'] = 'obj-updater %s' % os.getpid() try: with ConnectionTimeout(self.conn_timeout): conn = http_connect(node['ip'], node['port'], node['device'], - part, op, obj, headers) + part, op, obj, headers_out) with Timeout(self.node_timeout): resp = conn.getresponse() resp.read() diff --git a/swift/proxy/controllers/account.py b/swift/proxy/controllers/account.py index f3ff6458fc..d4fd4f6286 100644 --- a/swift/proxy/controllers/account.py +++ b/swift/proxy/controllers/account.py @@ -24,10 +24,9 @@ # These shenanigans are to ensure all related objects can be garbage # collected. We've seen objects hang around forever otherwise. -import time from urllib import unquote -from swift.common.utils import normalize_timestamp, public +from swift.common.utils import public from swift.common.constraints import check_metadata, MAX_ACCOUNT_NAME_LENGTH from swift.common.http import is_success, HTTP_NOT_FOUND from swift.proxy.controllers.base import Controller, get_account_memcache_key @@ -57,9 +56,7 @@ class AccountController(Controller): resp.body = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return resp - headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'X-Trans-Id': self.trans_id, - 'Connection': 'close'} + headers = self.generate_request_headers(req) resp = self.make_requests( Request.blank('/v1/' + self.account_name), self.app.account_ring, partition, 'PUT', @@ -90,10 +87,7 @@ class AccountController(Controller): return resp account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) - headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-trans-id': self.trans_id, - 'Connection': 'close'} - self.transfer_headers(req.headers, headers) + headers = self.generate_request_headers(req, transfer=True) if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) @@ -110,10 +104,7 @@ class AccountController(Controller): return error_response account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) - headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'X-Trans-Id': self.trans_id, - 'Connection': 'close'} - self.transfer_headers(req.headers, headers) + headers = self.generate_request_headers(req, transfer=True) if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) @@ -150,9 +141,7 @@ class AccountController(Controller): headers={'Allow': ', '.join(self.allowed_methods)}) account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) - headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'X-Trans-Id': self.trans_id, - 'Connection': 'close'} + headers = self.generate_request_headers(req) if self.app.memcache: self.app.memcache.delete( get_account_memcache_key(self.account_name)) diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 006514ed05..a7e12762e6 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -24,6 +24,7 @@ # These shenanigans are to ensure all related objects can be garbage # collected. We've seen objects hang around forever otherwise. +import os import time import functools import inspect @@ -42,7 +43,7 @@ from swift.common.http import is_informational, is_success, is_redirection, \ is_server_error, HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_MULTIPLE_CHOICES, \ HTTP_BAD_REQUEST, HTTP_NOT_FOUND, HTTP_SERVICE_UNAVAILABLE, \ HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED -from swift.common.swob import Request, Response +from swift.common.swob import Request, Response, HeaderKeyDict def update_headers(response, headers): @@ -178,8 +179,8 @@ def cors_validation(func): 'content-type', 'expires', 'last-modified', 'pragma', 'etag', 'x-timestamp', 'x-trans-id'] for header in resp.headers: - if header.startswith('x-container-meta') or \ - header.startswith('x-object-meta'): + if header.startswith('X-Container-Meta') or \ + header.startswith('X-Object-Meta'): expose_headers.append(header.lower()) if cors_info.get('expose_headers'): expose_headers.extend( @@ -280,20 +281,39 @@ class Controller(object): return [] def transfer_headers(self, src_headers, dst_headers): - st = self.server_type.lower() + x_remove = 'x-remove-%s-meta-' % st - x_meta = 'x-%s-meta-' % st dst_headers.update((k.lower().replace('-remove', '', 1), '') for k in src_headers if k.lower().startswith(x_remove) or k.lower() in self._x_remove_headers()) + x_meta = 'x-%s-meta-' % st dst_headers.update((k.lower(), v) for k, v in src_headers.iteritems() if k.lower() in self.pass_through_headers or k.lower().startswith(x_meta)) + def generate_request_headers(self, orig_req=None, additional=None, + transfer=False): + # Use the additional headers first so they don't overwrite the headers + # we require. + headers = HeaderKeyDict(additional) if additional else HeaderKeyDict() + if transfer: + self.transfer_headers(orig_req.headers, headers) + if 'x-timestamp' not in headers: + headers['x-timestamp'] = normalize_timestamp(time.time()) + if orig_req: + referer = orig_req.as_referer() + else: + referer = '' + headers.update({'x-trans-id': self.trans_id, + 'connection': 'close', + 'user-agent': 'proxy-server %s' % os.getpid(), + 'referer': referer}) + return headers + def error_occurred(self, node, msg): """ Handle logging, and handling of errors. @@ -359,11 +379,14 @@ class Controller(object): {'msg': msg, 'ip': node['ip'], 'port': node['port'], 'device': node['device']}) - def account_info(self, account, autocreate=False): + def account_info(self, account, req=None, autocreate=False): """ Get account information, and also verify that the account exists. :param account: name of the account to get the info for + :param req: caller's HTTP request context object (optional) + :param autocreate: whether or not to automatically create the given + account or not (optional, default: False) :returns: tuple of (account partition, account nodes, container_count) or (None, None, None) if it does not exist """ @@ -392,7 +415,7 @@ class Controller(object): return None, None, None result_code = 0 path = '/%s' % account - headers = {'x-trans-id': self.trans_id, 'Connection': 'close'} + headers = self.generate_request_headers(req) for node in self.iter_nodes(self.app.account_ring, partition): try: start_node_timing = time.time() @@ -432,9 +455,7 @@ class Controller(object): if result_code == HTTP_NOT_FOUND and autocreate: if len(account) > MAX_ACCOUNT_NAME_LENGTH: return None, None, None - headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'X-Trans-Id': self.trans_id, - 'Connection': 'close'} + headers = self.generate_request_headers(req) resp = self.make_requests(Request.blank('/v1' + path), self.app.account_ring, partition, 'PUT', path, [headers] * len(nodes)) @@ -460,7 +481,8 @@ class Controller(object): return partition, nodes, container_count return None, None, None - def container_info(self, account, container, account_autocreate=False): + def container_info(self, account, container, req=None, + account_autocreate=False): """ Get container information and thusly verify container existence. This will also make a call to account_info to verify that the @@ -468,6 +490,9 @@ class Controller(object): :param account: account name for the container :param container: container name to look up + :param req: caller's HTTP request context object (optional) + :param account_autocreate: whether or not to automatically create the + given account or not (optional, default: False) :returns: dict containing at least container partition ('partition'), container nodes ('containers'), container read acl ('read_acl'), container write acl ('write_acl'), @@ -492,9 +517,10 @@ class Controller(object): container_info['partition'] = part container_info['nodes'] = nodes return container_info - if not self.account_info(account, autocreate=account_autocreate)[1]: + if not self.account_info(account, req, + autocreate=account_autocreate)[1]: return container_info - headers = {'x-trans-id': self.trans_id, 'Connection': 'close'} + headers = self.generate_request_headers(req) for node in self.iter_nodes(self.app.container_ring, part): try: start_node_timing = time.time() @@ -784,8 +810,8 @@ class Controller(object): start_node_timing = time.time() try: with ConnectionTimeout(self.app.conn_timeout): - headers = dict(req.headers) - headers['Connection'] = 'close' + headers = self.generate_request_headers( + req, additional=req.headers) conn = http_connect( node['ip'], node['port'], node['device'], partition, req.method, path, headers=headers, diff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py index 83b56d7785..3229c69036 100644 --- a/swift/proxy/controllers/container.py +++ b/swift/proxy/controllers/container.py @@ -24,10 +24,9 @@ # These shenanigans are to ensure all related objects can be garbage # collected. We've seen objects hang around forever otherwise. -import time from urllib import unquote -from swift.common.utils import normalize_timestamp, public, csv_append +from swift.common.utils import public, csv_append from swift.common.constraints import check_metadata, MAX_CONTAINER_NAME_LENGTH from swift.common.http import HTTP_ACCEPTED from swift.proxy.controllers.base import Controller, delay_denial, \ @@ -70,7 +69,7 @@ class ContainerController(Controller): def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" - if not self.account_info(self.account_name)[1]: + if not self.account_info(self.account_name, req)[1]: return HTTPNotFound(request=req) part = self.app.container_ring.get_part( self.account_name, self.container_name) @@ -125,7 +124,7 @@ class ContainerController(Controller): (len(self.container_name), MAX_CONTAINER_NAME_LENGTH) return resp account_partition, accounts, container_count = \ - self.account_info(self.account_name, + self.account_info(self.account_name, req, autocreate=self.app.account_autocreate) if self.app.max_containers_per_account > 0 and \ container_count >= self.app.max_containers_per_account and \ @@ -158,16 +157,13 @@ class ContainerController(Controller): if error_response: return error_response account_partition, accounts, container_count = \ - self.account_info(self.account_name, + self.account_info(self.account_name, req, autocreate=self.app.account_autocreate) if not accounts: return HTTPNotFound(request=req) container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) - headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-trans-id': self.trans_id, - 'Connection': 'close'} - self.transfer_headers(req.headers, headers) + headers = self.generate_request_headers(req, transfer=True) if self.app.memcache: self.app.memcache.delete(get_container_memcache_key( self.account_name, self.container_name)) @@ -181,7 +177,7 @@ class ContainerController(Controller): def DELETE(self, req): """HTTP DELETE request handler.""" account_partition, accounts, container_count = \ - self.account_info(self.account_name) + self.account_info(self.account_name, req) if not accounts: return HTTPNotFound(request=req) container_partition, containers = self.app.container_ring.get_nodes( @@ -202,14 +198,9 @@ class ContainerController(Controller): def _backend_requests(self, req, n_outgoing, account_partition, accounts): - headers = [{'Connection': 'close', - 'X-Timestamp': normalize_timestamp(time.time()), - 'x-trans-id': self.trans_id} + headers = [self.generate_request_headers(req, transfer=True) for _junk in range(n_outgoing)] - for header in headers: - self.transfer_headers(req.headers, header) - for i, account in enumerate(accounts): i = i % len(headers) diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 1d8f7690f3..0c6454972b 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -377,8 +377,8 @@ class ObjectController(Controller): def GETorHEAD(self, req): """Handle HTTP GET or HEAD requests.""" - container_info = self.container_info(self.account_name, - self.container_name) + container_info = self.container_info( + self.account_name, self.container_name, req) req.acl = container_info['read_acl'] if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) @@ -568,7 +568,7 @@ class ObjectController(Controller): if error_response: return error_response container_info = self.container_info( - self.account_name, self.container_name, + self.account_name, self.container_name, req, account_autocreate=self.app.account_autocreate) container_partition = container_info['partition'] containers = container_info['nodes'] @@ -614,7 +614,7 @@ class ObjectController(Controller): def _backend_requests(self, req, n_outgoing, container_partition, containers, delete_at_partition=None, delete_at_nodes=None): - headers = [dict(req.headers.iteritems()) + headers = [self.generate_request_headers(req, additional=req.headers) for _junk in range(n_outgoing)] for header in headers: @@ -692,7 +692,7 @@ class ObjectController(Controller): def PUT(self, req): """HTTP PUT request handler.""" container_info = self.container_info( - self.account_name, self.container_name, + self.account_name, self.container_name, req, account_autocreate=self.app.account_autocreate) container_partition = container_info['partition'] containers = container_info['nodes'] @@ -991,8 +991,8 @@ class ObjectController(Controller): @delay_denial def DELETE(self, req): """HTTP DELETE request handler.""" - container_info = self.container_info(self.account_name, - self.container_name) + container_info = self.container_info( + self.account_name, self.container_name, req) container_partition = container_info['partition'] containers = container_info['nodes'] req.acl = container_info['write_acl'] @@ -1043,8 +1043,8 @@ class ObjectController(Controller): self.container_name = lcontainer self.object_name = last_item['name'] new_del_req = Request.blank(copy_path, environ=req.environ) - container_info = self.container_info(self.account_name, - self.container_name) + container_info = self.container_info( + self.account_name, self.container_name, req) container_partition = container_info['partition'] containers = container_info['nodes'] new_del_req.acl = container_info['write_acl'] diff --git a/test/unit/common/middleware/test_except.py b/test/unit/common/middleware/test_except.py index 0295ae1697..3efb8b194c 100644 --- a/test/unit/common/middleware/test_except.py +++ b/test/unit/common/middleware/test_except.py @@ -59,7 +59,7 @@ class TestCatchErrors(unittest.TestCase): self.assertEquals(self.logger.txn_id, None) def start_response(status, headers, exc_info=None): - self.assert_('x-trans-id' in (x[0] for x in headers)) + self.assert_('X-Trans-Id' in (x[0] for x in headers)) app = catch_errors.CatchErrorMiddleware(FakeApp(), {}) req = Request.blank('/v1/a/c/o') app(req.environ, start_response) @@ -69,7 +69,7 @@ class TestCatchErrors(unittest.TestCase): self.assertEquals(self.logger.txn_id, None) def start_response(status, headers, exc_info=None): - self.assert_('x-trans-id' in (x[0] for x in headers)) + self.assert_('X-Trans-Id' in (x[0] for x in headers)) app = catch_errors.CatchErrorMiddleware(FakeApp(True), {}) req = Request.blank('/v1/a/c/o') app(req.environ, start_response) @@ -86,7 +86,7 @@ class TestCatchErrors(unittest.TestCase): self.assertEquals(self.logger.txn_id, None) def start_response(status, headers, exc_info=None): - self.assert_('x-trans-id' in (x[0] for x in headers)) + self.assert_('X-Trans-Id' in (x[0] for x in headers)) app = catch_errors.CatchErrorMiddleware( FakeApp(), {'trans_id_suffix': '-stuff'}) req = Request.blank('/v1/a/c/o') diff --git a/test/unit/common/middleware/test_tempurl.py b/test/unit/common/middleware/test_tempurl.py index fd58f506c4..127c970bfb 100644 --- a/test/unit/common/middleware/test_tempurl.py +++ b/test/unit/common/middleware/test_tempurl.py @@ -19,7 +19,7 @@ from hashlib import sha1 from contextlib import contextmanager from time import time -from swift.common.swob import Request, Response +from swift.common.swob import Request, Response, HeaderKeyDict from swift.common.middleware import tempauth, tempurl @@ -706,7 +706,7 @@ class TestTempURL(unittest.TestCase): orh = '' oah = '' hdrs = {'test-header': 'value'} - hdrs = dict(tempurl.TempURL(None, + hdrs = HeaderKeyDict(tempurl.TempURL(None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.iteritems())) self.assertTrue('test-header' in hdrs) @@ -714,7 +714,7 @@ class TestTempURL(unittest.TestCase): orh = 'test-header' oah = '' hdrs = {'test-header': 'value'} - hdrs = dict(tempurl.TempURL(None, + hdrs = HeaderKeyDict(tempurl.TempURL(None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.iteritems())) self.assertTrue('test-header' not in hdrs) @@ -723,7 +723,7 @@ class TestTempURL(unittest.TestCase): oah = '' hdrs = {'test-header-one': 'value', 'test-header-two': 'value'} - hdrs = dict(tempurl.TempURL(None, + hdrs = HeaderKeyDict(tempurl.TempURL(None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.iteritems())) self.assertTrue('test-header-one' not in hdrs) @@ -733,7 +733,7 @@ class TestTempURL(unittest.TestCase): oah = 'test-header-two' hdrs = {'test-header-one': 'value', 'test-header-two': 'value'} - hdrs = dict(tempurl.TempURL(None, + hdrs = HeaderKeyDict(tempurl.TempURL(None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.iteritems())) self.assertTrue('test-header-one' not in hdrs) @@ -746,7 +746,7 @@ class TestTempURL(unittest.TestCase): 'test-other-header': 'value', 'test-header-yes': 'value', 'test-header-yes-this': 'value'} - hdrs = dict(tempurl.TempURL(None, + hdrs = HeaderKeyDict(tempurl.TempURL(None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.iteritems())) self.assertTrue('test-header-one' not in hdrs) diff --git a/test/unit/common/test_direct_client.py b/test/unit/common/test_direct_client.py index d1996e58f7..7bcfc705ec 100644 --- a/test/unit/common/test_direct_client.py +++ b/test/unit/common/test_direct_client.py @@ -16,14 +16,42 @@ # TODO: Tests import unittest +import os from swift.common import direct_client class TestDirectClient(unittest.TestCase): - def test_placeholder(self): - pass + def test_quote(self): + res = direct_client.quote('123') + assert res == '123' + res = direct_client.quote('1&2&/3') + assert res == '1%262%26/3' + res = direct_client.quote('1&2&3', safe='&') + assert res == '1&2&3' + + def test_gen_headers(self): + hdrs = direct_client.gen_headers() + assert 'user-agent' in hdrs + assert hdrs['user-agent'] == 'direct-client %s' % os.getpid() + assert len(hdrs.keys()) == 1 + + hdrs = direct_client.gen_headers(add_ts=True) + assert 'user-agent' in hdrs + assert 'x-timestamp' in hdrs + assert len(hdrs.keys()) == 2 + + hdrs = direct_client.gen_headers(hdrs_in={'foo-bar': '47'}) + assert 'user-agent' in hdrs + assert 'foo-bar' in hdrs + assert hdrs['foo-bar'] == '47' + assert len(hdrs.keys()) == 2 + + hdrs = direct_client.gen_headers(hdrs_in={'user-agent': '47'}) + assert 'user-agent' in hdrs + assert hdrs['user-agent'] == 'direct-client %s' % os.getpid() + assert len(hdrs.keys()) == 1 if __name__ == '__main__': diff --git a/test/unit/common/test_swob.py b/test/unit/common/test_swob.py index d20f225d39..44b4798a3d 100644 --- a/test/unit/common/test_swob.py +++ b/test/unit/common/test_swob.py @@ -102,6 +102,15 @@ class TestHeaderKeyDict(unittest.TestCase): self.assertEquals(headers.get('something-else'), None) self.assertEquals(headers.get('something-else', True), True) + def test_keys(self): + headers = swift.common.swob.HeaderKeyDict() + headers['content-length'] = 20 + headers['cOnTent-tYpe'] = 'text/plain' + headers['SomeThing-eLse'] = 'somevalue' + self.assertEquals( + set(headers.keys()), + set(('Content-Length', 'Content-Type', 'Something-Else'))) + class TestRange(unittest.TestCase): def test_range(self): @@ -502,6 +511,61 @@ class TestRequest(unittest.TestCase): req.query_string = u'x=\u2661' self.assertEquals(req.params['x'], u'\u2661'.encode('utf-8')) + def test_url(self): + pi = '/hi/there' + path = pi + req = swift.common.swob.Request.blank(path) + sche = 'http' + exp_url = '%(sche)s://localhost%(pi)s' % locals() + self.assertEqual(req.url, exp_url) + + qs = 'hello=equal&acl' + path = '%s?%s' % (pi, qs) + s, p = 'unit.test.example.com', '90' + req = swift.common.swob.Request({'PATH_INFO': pi, + 'QUERY_STRING': qs, + 'SERVER_NAME': s, + 'SERVER_PORT': p}) + exp_url = '%(sche)s://%(s)s:%(p)s%(pi)s?%(qs)s' % locals() + self.assertEqual(req.url, exp_url) + + host = 'unit.test.example.com' + req = swift.common.swob.Request({'PATH_INFO': pi, + 'QUERY_STRING': qs, + 'HTTP_HOST': host + ':80'}) + exp_url = '%(sche)s://%(host)s%(pi)s?%(qs)s' % locals() + self.assertEqual(req.url, exp_url) + + host = 'unit.test.example.com' + sche = 'https' + req = swift.common.swob.Request({'PATH_INFO': pi, + 'QUERY_STRING': qs, + 'HTTP_HOST': host + ':443', + 'wsgi.url_scheme': sche}) + exp_url = '%(sche)s://%(host)s%(pi)s?%(qs)s' % locals() + self.assertEqual(req.url, exp_url) + + host = 'unit.test.example.com:81' + req = swift.common.swob.Request({'PATH_INFO': pi, + 'QUERY_STRING': qs, + 'HTTP_HOST': host, + 'wsgi.url_scheme': sche}) + exp_url = '%(sche)s://%(host)s%(pi)s?%(qs)s' % locals() + self.assertEqual(req.url, exp_url) + + def test_as_referer(self): + pi = '/hi/there' + qs = 'hello=equal&acl' + sche = 'https' + host = 'unit.test.example.com:81' + req = swift.common.swob.Request({'REQUEST_METHOD': 'POST', + 'PATH_INFO': pi, + 'QUERY_STRING': qs, + 'HTTP_HOST': host, + 'wsgi.url_scheme': sche}) + exp_url = '%(sche)s://%(host)s%(pi)s?%(qs)s' % locals() + self.assertEqual(req.as_referer(), 'POST ' + exp_url) + class TestStatusMap(unittest.TestCase): def test_status_map(self): @@ -518,8 +582,8 @@ class TestStatusMap(unittest.TestCase): self.assert_('The resource could not be found.' in body) self.assertEquals(response_args[0], '404 Not Found') headers = dict(response_args[1]) - self.assertEquals(headers['content-type'], 'text/html; charset=UTF-8') - self.assert_(int(headers['content-length']) > 0) + self.assertEquals(headers['Content-Type'], 'text/html; charset=UTF-8') + self.assert_(int(headers['Content-Length']) > 0) class TestResponse(unittest.TestCase): diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index ca3f860d8a..60e58b22c1 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -24,7 +24,7 @@ from tempfile import mkdtemp from eventlet import spawn, Timeout, listen import simplejson -from swift.common.swob import Request +from swift.common.swob import Request, HeaderKeyDict import swift.container from swift.container import server as container_server from swift.common.utils import normalize_timestamp, mkdirs @@ -1353,11 +1353,13 @@ class TestContainerController(unittest.TestCase): 'partition': '30', 'method': 'PUT', 'ssl': False, - 'headers': {'x-bytes-used': 0, + 'headers': HeaderKeyDict({'x-bytes-used': 0, 'x-delete-timestamp': '0', 'x-object-count': 0, 'x-put-timestamp': '0000012345.00000', - 'x-trans-id': '-'}}) + 'referer': 'PUT http://localhost/sda1/p/a/c', + 'user-agent': 'container-server %d' % os.getpid(), + 'x-trans-id': '-'})}) self.assertEquals( http_connect_args[1], {'ipaddr': '6.7.8.9', @@ -1367,11 +1369,13 @@ class TestContainerController(unittest.TestCase): 'partition': '30', 'method': 'PUT', 'ssl': False, - 'headers': {'x-bytes-used': 0, + 'headers': HeaderKeyDict({'x-bytes-used': 0, 'x-delete-timestamp': '0', 'x-object-count': 0, 'x-put-timestamp': '0000012345.00000', - 'x-trans-id': '-'}}) + 'referer': 'PUT http://localhost/sda1/p/a/c', + 'user-agent': 'container-server %d' % os.getpid(), + 'x-trans-id': '-'})}) if __name__ == '__main__': diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index a74738b6c5..b6afe34ece 100755 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -37,7 +37,7 @@ from swift.common.utils import hash_path, mkdirs, normalize_timestamp, \ from swift.common.exceptions import DiskFileNotExist from swift.common import constraints from eventlet import tpool -from swift.common.swob import Request +from swift.common.swob import Request, HeaderKeyDict class TestDiskFile(unittest.TestCase): @@ -1705,7 +1705,8 @@ class TestObjectController(unittest.TestCase): finally: object_server.http_connect = orig_http_connect self.assertEquals(given_args, ['127.0.0.1', '1234', 'sdc1', 1, 'PUT', - '/a/c/o', {'x-timestamp': '1', 'x-out': 'set'}]) + '/a/c/o', {'x-timestamp': '1', 'x-out': 'set', + 'user-agent': 'obj-server %s' % os.getpid()}]) def test_updating_multiple_delete_at_container_servers(self): @@ -1771,11 +1772,13 @@ class TestObjectController(unittest.TestCase): 'partition': '20', 'method': 'PUT', 'ssl': False, - 'headers': {'x-content-type': 'application/burrito', + 'headers': HeaderKeyDict({'x-content-type': 'application/burrito', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', 'x-timestamp': '12345', - 'x-trans-id': '-'}}) + 'referer': 'PUT http://localhost/sda1/p/a/c/o', + 'user-agent': 'obj-server %d' % os.getpid(), + 'x-trans-id': '-'})}) self.assertEquals( http_connect_args[1], {'ipaddr': '10.1.1.1', @@ -1785,11 +1788,13 @@ class TestObjectController(unittest.TestCase): 'partition': '6237', 'method': 'PUT', 'ssl': False, - 'headers': {'x-content-type': 'text/plain', + 'headers': HeaderKeyDict({'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', 'x-timestamp': '12345', - 'x-trans-id': '-'}}) + 'referer': 'PUT http://localhost/sda1/p/a/c/o', + 'user-agent': 'obj-server %d' % os.getpid(), + 'x-trans-id': '-'})}) self.assertEquals( http_connect_args[2], {'ipaddr': '10.2.2.2', @@ -1799,11 +1804,13 @@ class TestObjectController(unittest.TestCase): 'partition': '6237', 'method': 'PUT', 'ssl': False, - 'headers': {'x-content-type': 'text/plain', + 'headers': HeaderKeyDict({'x-content-type': 'text/plain', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', 'x-timestamp': '12345', - 'x-trans-id': '-'}}) + 'referer': 'PUT http://localhost/sda1/p/a/c/o', + 'user-agent': 'obj-server %d' % os.getpid(), + 'x-trans-id': '-'})}) def test_updating_multiple_container_servers(self): http_connect_args = [] @@ -1858,11 +1865,13 @@ class TestObjectController(unittest.TestCase): 'partition': '20', 'method': 'PUT', 'ssl': False, - 'headers': {'x-content-type': 'application/burrito', + 'headers': HeaderKeyDict({'x-content-type': 'application/burrito', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', 'x-timestamp': '12345', - 'x-trans-id': '-'}}) + 'referer': 'PUT http://localhost/sda1/p/a/c/o', + 'user-agent': 'obj-server %d' % os.getpid(), + 'x-trans-id': '-'})}) self.assertEquals( http_connect_args[1], {'ipaddr': '6.7.8.9', @@ -1872,11 +1881,13 @@ class TestObjectController(unittest.TestCase): 'partition': '20', 'method': 'PUT', 'ssl': False, - 'headers': {'x-content-type': 'application/burrito', + 'headers': HeaderKeyDict({'x-content-type': 'application/burrito', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-size': '0', 'x-timestamp': '12345', - 'x-trans-id': '-'}}) + 'referer': 'PUT http://localhost/sda1/p/a/c/o', + 'user-agent': 'obj-server %d' % os.getpid(), + 'x-trans-id': '-'})}) def test_async_update_saves_on_exception(self): _prefix = utils.HASH_PATH_PREFIX @@ -1898,8 +1909,9 @@ class TestObjectController(unittest.TestCase): pickle.load(open(os.path.join(self.testdir, 'sda1', 'async_pending', 'a83', '06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))), - {'headers': {'x-timestamp': '1', 'x-out': 'set'}, 'account': 'a', - 'container': 'c', 'obj': 'o', 'op': 'PUT'}) + {'headers': {'x-timestamp': '1', 'x-out': 'set', + 'user-agent': 'obj-server %s' % os.getpid()}, + 'account': 'a', 'container': 'c', 'obj': 'o', 'op': 'PUT'}) def test_async_update_saves_on_non_2xx(self): _prefix = utils.HASH_PATH_PREFIX @@ -1931,7 +1943,8 @@ class TestObjectController(unittest.TestCase): pickle.load(open(os.path.join(self.testdir, 'sda1', 'async_pending', 'a83', '06fbf0b514e5199dfc4e00f42eb5ea83-0000000001.00000'))), - {'headers': {'x-timestamp': '1', 'x-out': str(status)}, + {'headers': {'x-timestamp': '1', 'x-out': str(status), + 'user-agent': 'obj-server %s' % os.getpid()}, 'account': 'a', 'container': 'c', 'obj': 'o', 'op': 'PUT'}) finally: @@ -1969,6 +1982,46 @@ class TestObjectController(unittest.TestCase): finally: object_server.http_connect = orig_http_connect + def test_container_update_no_async_update(self): + given_args = [] + + def fake_async_update(*args): + given_args.extend(args) + + self.object_controller.async_update = fake_async_update + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': 1, + 'X-Trans-Id': '1234'}) + self.object_controller.container_update('PUT', 'a', 'c', 'o', req, + {'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', + 'x-content-type': 'text/plain', 'x-timestamp': '1'}, 'sda1') + self.assertEquals(given_args, []) + + def test_container_update(self): + given_args = [] + + def fake_async_update(*args): + given_args.extend(args) + + self.object_controller.async_update = fake_async_update + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': 1, + 'X-Trans-Id': '123', + 'X-Container-Host': 'chost', + 'X-Container-Partition': 'cpartition', + 'X-Container-Device': 'cdevice'}) + self.object_controller.container_update('PUT', 'a', 'c', 'o', req, + {'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', + 'x-content-type': 'text/plain', 'x-timestamp': '1'}, 'sda1') + self.assertEquals(given_args, ['PUT', 'a', 'c', 'o', 'chost', + 'cpartition', 'cdevice', + {'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', + 'x-content-type': 'text/plain', 'x-timestamp': '1', + 'x-trans-id': '123', 'referer': 'PUT http://localhost/v1/a/c/o'}, + 'sda1']) + def test_delete_at_update_put(self): given_args = [] @@ -1976,13 +2029,18 @@ class TestObjectController(unittest.TestCase): given_args.extend(args) self.object_controller.async_update = fake_async_update + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': 1, + 'X-Trans-Id': '123'}) self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o', - {'x-timestamp': '1'}, 'sda1') + req, 'sda1') self.assertEquals(given_args, ['PUT', '.expiring_objects', '0', '2-a/c/o', None, None, None, - {'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', + HeaderKeyDict({'x-size': '0', + 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', 'x-timestamp': '1', - 'x-trans-id': '-'}, + 'x-trans-id': '123', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1']) def test_delete_at_negative(self): @@ -1993,13 +2051,18 @@ class TestObjectController(unittest.TestCase): given_args.extend(args) self.object_controller.async_update = fake_async_update + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': 1, + 'X-Trans-Id': '1234'}) self.object_controller.delete_at_update( - 'PUT', -2, 'a', 'c', 'o', {'x-timestamp': '1'}, 'sda1') + 'PUT', -2, 'a', 'c', 'o', req, 'sda1') self.assertEquals(given_args, [ 'PUT', '.expiring_objects', '0', '0-a/c/o', None, None, None, - {'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', + HeaderKeyDict({'x-size': '0', + 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', 'x-timestamp': '1', - 'x-trans-id': '-'}, + 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1']) def test_delete_at_cap(self): @@ -2010,14 +2073,19 @@ class TestObjectController(unittest.TestCase): given_args.extend(args) self.object_controller.async_update = fake_async_update + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': 1, + 'X-Trans-Id': '1234'}) self.object_controller.delete_at_update( - 'PUT', 12345678901, 'a', 'c', 'o', {'x-timestamp': '1'}, 'sda1') + 'PUT', 12345678901, 'a', 'c', 'o', req, 'sda1') self.assertEquals(given_args, [ 'PUT', '.expiring_objects', '9999936000', '9999999999-a/c/o', None, None, None, - {'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', + HeaderKeyDict({'x-size': '0', + 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', 'x-timestamp': '1', - 'x-trans-id': '-'}, + 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1']) def test_delete_at_update_put_with_info(self): @@ -2027,15 +2095,21 @@ class TestObjectController(unittest.TestCase): given_args.extend(args) self.object_controller.async_update = fake_async_update + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': 1, + 'X-Trans-Id': '1234', + 'X-Delete-At-Host': '127.0.0.1:1234', + 'X-Delete-At-Partition': '3', + 'X-Delete-At-Device': 'sdc1'}) self.object_controller.delete_at_update('PUT', 2, 'a', 'c', 'o', - {'x-timestamp': '1', 'X-Delete-At-Host': '127.0.0.1:1234', - 'X-Delete-At-Partition': '3', 'X-Delete-At-Device': 'sdc1'}, - 'sda1') + req, 'sda1') self.assertEquals(given_args, ['PUT', '.expiring_objects', '0', '2-a/c/o', '127.0.0.1:1234', '3', 'sdc1', - {'x-size': '0', 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', + HeaderKeyDict({'x-size': '0', + 'x-etag': 'd41d8cd98f00b204e9800998ecf8427e', 'x-content-type': 'text/plain', 'x-timestamp': '1', - 'x-trans-id': '-'}, + 'x-trans-id': '1234', 'referer': 'PUT http://localhost/v1/a/c/o'}), 'sda1']) def test_delete_at_update_delete(self): @@ -2045,11 +2119,16 @@ class TestObjectController(unittest.TestCase): given_args.extend(args) self.object_controller.async_update = fake_async_update + req = Request.blank('/v1/a/c/o', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'X-Timestamp': 1, + 'X-Trans-Id': '1234'}) self.object_controller.delete_at_update('DELETE', 2, 'a', 'c', 'o', - {'x-timestamp': '1'}, 'sda1') + req, 'sda1') self.assertEquals(given_args, ['DELETE', '.expiring_objects', '0', '2-a/c/o', None, None, None, - {'x-timestamp': '1', 'x-trans-id': '-'}, 'sda1']) + HeaderKeyDict({'x-timestamp': '1', 'x-trans-id': '1234', + 'referer': 'DELETE http://localhost/v1/a/c/o'}), 'sda1']) def test_POST_calls_delete_at(self): given_args = [] @@ -2089,11 +2168,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 202) self.assertEquals(given_args, [ 'PUT', int(delete_at_timestamp1), 'a', 'c', 'o', - {'X-Delete-At': delete_at_timestamp1, - 'Content-Type': 'application/x-test', - 'X-Timestamp': timestamp1, - 'Host': 'localhost:80'}, - 'sda1']) + req, 'sda1']) while given_args: given_args.pop() @@ -2110,18 +2185,9 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 202) self.assertEquals(given_args, [ 'PUT', int(delete_at_timestamp2), 'a', 'c', 'o', - {'X-Delete-At': delete_at_timestamp2, - 'Content-Type': 'application/x-test', - 'X-Timestamp': timestamp2, 'Host': 'localhost:80'}, - 'sda1', + req, 'sda1', 'DELETE', int(delete_at_timestamp1), 'a', 'c', 'o', - # This 2 timestamp is okay because it's ignored since it's just - # part of the current request headers. The above 1 timestamp is the - # important one. - {'X-Delete-At': delete_at_timestamp2, - 'Content-Type': 'application/x-test', - 'X-Timestamp': timestamp2, 'Host': 'localhost:80'}, - 'sda1']) + req, 'sda1']) def test_PUT_calls_delete_at(self): given_args = [] @@ -2153,12 +2219,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(given_args, [ 'PUT', int(delete_at_timestamp1), 'a', 'c', 'o', - {'X-Delete-At': delete_at_timestamp1, - 'Content-Length': '4', - 'Content-Type': 'application/octet-stream', - 'X-Timestamp': timestamp1, - 'Host': 'localhost:80'}, - 'sda1']) + req, 'sda1']) while given_args: given_args.pop() @@ -2177,20 +2238,9 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(given_args, [ 'PUT', int(delete_at_timestamp2), 'a', 'c', 'o', - {'X-Delete-At': delete_at_timestamp2, - 'Content-Length': '4', - 'Content-Type': 'application/octet-stream', - 'X-Timestamp': timestamp2, 'Host': 'localhost:80'}, - 'sda1', + req, 'sda1', 'DELETE', int(delete_at_timestamp1), 'a', 'c', 'o', - # This 2 timestamp is okay because it's ignored since it's just - # part of the current request headers. The above 1 timestamp is the - # important one. - {'X-Delete-At': delete_at_timestamp2, - 'Content-Length': '4', - 'Content-Type': 'application/octet-stream', - 'X-Timestamp': timestamp2, 'Host': 'localhost:80'}, - 'sda1']) + req, 'sda1']) def test_GET_but_expired(self): test_time = time() + 10000 @@ -2434,12 +2484,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 201) self.assertEquals(given_args, [ 'PUT', int(delete_at_timestamp1), 'a', 'c', 'o', - {'X-Delete-At': delete_at_timestamp1, - 'Content-Length': '4', - 'Content-Type': 'application/octet-stream', - 'X-Timestamp': timestamp1, - 'Host': 'localhost:80'}, - 'sda1']) + req, 'sda1']) while given_args: given_args.pop() @@ -2454,9 +2499,7 @@ class TestObjectController(unittest.TestCase): self.assertEquals(resp.status_int, 204) self.assertEquals(given_args, [ 'DELETE', int(delete_at_timestamp1), 'a', 'c', 'o', - {'Content-Type': 'application/octet-stream', - 'Host': 'localhost:80', 'X-Timestamp': timestamp2}, - 'sda1']) + req, 'sda1']) def test_PUT_delete_at_in_past(self): req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 5da6cc4078..15eefec860 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -167,7 +167,7 @@ def setup(): fd.flush() headers = readuntil2crlfs(fd) exp = 'HTTP/1.1 201' - assert(headers[:len(exp)] == exp) + assert headers[:len(exp)] == exp, "Expected '%s', encountered '%s'" % (exp, headers[:len(exp)]) def teardown(): @@ -311,8 +311,17 @@ class TestController(unittest.TestCase): object_ring=FakeRing()) self.controller = swift.proxy.controllers.Controller(app) + class FakeReq(object): + def __init__(self): + self.url = "/foo/bar" + self.method = "METHOD" + + def as_referer(self): + return self.method + ' ' + self.url + self.account = 'some_account' self.container = 'some_container' + self.request = FakeReq() self.read_acl = 'read_acl' self.write_acl = 'write_acl' @@ -365,7 +374,7 @@ class TestController(unittest.TestCase): with save_globals(): set_http_connect(200) partition, nodes, count = \ - self.controller.account_info(self.account) + self.controller.account_info(self.account, self.request) set_http_connect(201, raise_timeout_exc=True) self.controller._make_request( nodes, partition, 'POST', '/', '', '', @@ -376,7 +385,7 @@ class TestController(unittest.TestCase): with save_globals(): set_http_connect(200) partition, nodes, count = \ - self.controller.account_info(self.account) + self.controller.account_info(self.account, self.request) self.check_account_info_return(partition, nodes) self.assertEquals(count, 12345) @@ -391,7 +400,7 @@ class TestController(unittest.TestCase): set_http_connect() partition, nodes, count = \ - self.controller.account_info(self.account) + self.controller.account_info(self.account, self.request) self.check_account_info_return(partition, nodes) self.assertEquals(count, 12345) @@ -400,7 +409,7 @@ class TestController(unittest.TestCase): with save_globals(): set_http_connect(404, 404, 404) partition, nodes, count = \ - self.controller.account_info(self.account) + self.controller.account_info(self.account, self.request) self.check_account_info_return(partition, nodes, True) self.assertEquals(count, None) @@ -415,7 +424,7 @@ class TestController(unittest.TestCase): set_http_connect() partition, nodes, count = \ - self.controller.account_info(self.account) + self.controller.account_info(self.account, self.request) self.check_account_info_return(partition, nodes, True) self.assertEquals(count, None) @@ -424,7 +433,7 @@ class TestController(unittest.TestCase): def test(*status_list): set_http_connect(*status_list) partition, nodes, count = \ - self.controller.account_info(self.account) + self.controller.account_info(self.account, self.request) self.assertEqual(len(self.memcache.keys()), 0) self.check_account_info_return(partition, nodes, True) self.assertEquals(count, None) @@ -442,28 +451,28 @@ class TestController(unittest.TestCase): # is True set_http_connect(404, 404, 404) partition, nodes, count = \ - self.controller.account_info(self.account, autocreate=False) + self.controller.account_info(self.account, self.request, autocreate=False) self.check_account_info_return(partition, nodes, is_none=True) self.assertEquals(count, None) self.memcache.store = {} set_http_connect(404, 404, 404) partition, nodes, count = \ - self.controller.account_info(self.account) + self.controller.account_info(self.account, self.request) self.check_account_info_return(partition, nodes, is_none=True) self.assertEquals(count, None) self.memcache.store = {} set_http_connect(404, 404, 404, 201, 201, 201) partition, nodes, count = \ - self.controller.account_info(self.account, autocreate=True) + self.controller.account_info(self.account, self.request, autocreate=True) self.check_account_info_return(partition, nodes) self.assertEquals(count, 0) self.memcache.store = {} set_http_connect(404, 404, 404, 503, 201, 201) partition, nodes, count = \ - self.controller.account_info(self.account, autocreate=True) + self.controller.account_info(self.account, self.request, autocreate=True) self.check_account_info_return(partition, nodes) self.assertEquals(count, 0) @@ -471,7 +480,7 @@ class TestController(unittest.TestCase): set_http_connect(404, 404, 404, 503, 201, 503) exc = None partition, nodes, count = \ - self.controller.account_info(self.account, autocreate=True) + self.controller.account_info(self.account, self.request, autocreate=True) self.check_account_info_return(partition, nodes, is_none=True) self.assertEquals(None, count) @@ -479,7 +488,7 @@ class TestController(unittest.TestCase): set_http_connect(404, 404, 404, 403, 403, 403) exc = None partition, nodes, count = \ - self.controller.account_info(self.account, autocreate=True) + self.controller.account_info(self.account, self.request, autocreate=True) self.check_account_info_return(partition, nodes, is_none=True) self.assertEquals(None, count) @@ -487,7 +496,7 @@ class TestController(unittest.TestCase): set_http_connect(404, 404, 404, 409, 409, 409) exc = None partition, nodes, count = \ - self.controller.account_info(self.account, autocreate=True) + self.controller.account_info(self.account, self.request, autocreate=True) self.check_account_info_return(partition, nodes, is_none=True) self.assertEquals(None, count) @@ -504,18 +513,19 @@ class TestController(unittest.TestCase): self.assertEqual(write_acl, ret['write_acl']) def test_container_info_invalid_account(self): - def account_info(self, account, autocreate=False): + def account_info(self, account, request, autocreate=False): return None, None with save_globals(): swift.proxy.controllers.Controller.account_info = account_info ret = self.controller.container_info(self.account, - self.container) + self.container, + self.request) self.check_container_info_return(ret, True) # tests if 200 is cached and used def test_container_info_200(self): - def account_info(self, account, autocreate=False): + def account_info(self, account, request, autocreate=False): return True, True, 0 with save_globals(): @@ -523,8 +533,8 @@ class TestController(unittest.TestCase): 'x-container-write': self.write_acl} swift.proxy.controllers.Controller.account_info = account_info set_http_connect(200, headers=headers) - ret = self.controller.container_info(self.account, - self.container) + ret = self.controller.container_info( + self.account, self.container, self.request) self.check_container_info_return(ret) cache_key = get_container_memcache_key(self.account, @@ -534,20 +544,20 @@ class TestController(unittest.TestCase): self.assertEquals(200, cache_value.get('status')) set_http_connect() - ret = self.controller.container_info(self.account, - self.container) + ret = self.controller.container_info( + self.account, self.container, self.request) self.check_container_info_return(ret) # tests if 404 is cached and used def test_container_info_404(self): - def account_info(self, account, autocreate=False): + def account_info(self, account, request, autocreate=False): return True, True, 0 with save_globals(): swift.proxy.controllers.Controller.account_info = account_info set_http_connect(404, 404, 404) - ret = self.controller.container_info(self.account, - self.container) + ret = self.controller.container_info( + self.account, self.container, self.request) self.check_container_info_return(ret, True) cache_key = get_container_memcache_key(self.account, @@ -557,16 +567,16 @@ class TestController(unittest.TestCase): self.assertEquals(404, cache_value.get('status')) set_http_connect() - ret = self.controller.container_info(self.account, - self.container) + ret = self.controller.container_info( + self.account, self.container, self.request) self.check_container_info_return(ret, True) # tests if some http status codes are not cached def test_container_info_no_cache(self): def test(*status_list): set_http_connect(*status_list) - ret = self.controller.container_info(self.account, - self.container) + ret = self.controller.container_info( + self.account, self.container, self.request) self.assertEqual(len(self.memcache.keys()), 0) self.check_container_info_return(ret, True) @@ -4303,13 +4313,13 @@ class TestObjectController(unittest.TestCase): 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT self.assertEqual(seen_headers, [ {'X-Container-Host': '10.0.0.0:1000', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sda'}, {'X-Container-Host': '10.0.0.1:1001', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sdb'}, {'X-Container-Host': '10.0.0.2:1002', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sdc'}]) def test_PUT_x_container_headers_with_fewer_container_replicas(self): @@ -4324,10 +4334,10 @@ class TestObjectController(unittest.TestCase): self.assertEqual(seen_headers, [ {'X-Container-Host': '10.0.0.0:1000', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sda'}, {'X-Container-Host': '10.0.0.1:1001', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sdb'}, {'X-Container-Host': None, 'X-Container-Partition': None, @@ -4345,13 +4355,13 @@ class TestObjectController(unittest.TestCase): self.assertEqual(seen_headers, [ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sda,sdd'}, {'X-Container-Host': '10.0.0.1:1001', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sdb'}, {'X-Container-Host': '10.0.0.2:1002', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sdc'}]) def test_POST_x_container_headers_with_more_container_replicas(self): @@ -4367,13 +4377,13 @@ class TestObjectController(unittest.TestCase): self.assertEqual(seen_headers, [ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sda,sdd'}, {'X-Container-Host': '10.0.0.1:1001', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sdb'}, {'X-Container-Host': '10.0.0.2:1002', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sdc'}]) def test_DELETE_x_container_headers_with_more_container_replicas(self): @@ -4388,13 +4398,13 @@ class TestObjectController(unittest.TestCase): self.assertEqual(seen_headers, [ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sda,sdd'}, {'X-Container-Host': '10.0.0.1:1001', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sdb'}, {'X-Container-Host': '10.0.0.2:1002', - 'X-Container-Partition': 1, + 'X-Container-Partition': '1', 'X-Container-Device': 'sdc'}]) def test_PUT_x_delete_at_with_fewer_container_replicas(self): @@ -4413,10 +4423,10 @@ class TestObjectController(unittest.TestCase): self.assertEqual(seen_headers, [ {'X-Delete-At-Host': '10.0.0.0:1000', - 'X-Delete-At-Partition': 1, + 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sda'}, {'X-Delete-At-Host': '10.0.0.1:1001', - 'X-Delete-At-Partition': 1, + 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sdb'}, {'X-Delete-At-Host': None, 'X-Delete-At-Partition': None, @@ -4439,13 +4449,13 @@ class TestObjectController(unittest.TestCase): 'X-Delete-At-Partition')) self.assertEqual(seen_headers, [ {'X-Delete-At-Host': '10.0.0.0:1000,10.0.0.3:1003', - 'X-Delete-At-Partition': 1, + 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sda,sdd'}, {'X-Delete-At-Host': '10.0.0.1:1001', - 'X-Delete-At-Partition': 1, + 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sdb'}, {'X-Delete-At-Host': '10.0.0.2:1002', - 'X-Delete-At-Partition': 1, + 'X-Delete-At-Partition': '1', 'X-Delete-At-Device': 'sdc'}]) @@ -5189,10 +5199,10 @@ class TestContainerController(unittest.TestCase): 200, 201, 201, 201) # HEAD PUT PUT PUT self.assertEqual(seen_headers, [ {'X-Account-Host': '10.0.0.0:1000', - 'X-Account-Partition': 1, + 'X-Account-Partition': '1', 'X-Account-Device': 'sda'}, {'X-Account-Host': '10.0.0.1:1001', - 'X-Account-Partition': 1, + 'X-Account-Partition': '1', 'X-Account-Device': 'sdb'}, {'X-Account-Host': None, 'X-Account-Partition': None, @@ -5208,13 +5218,13 @@ class TestContainerController(unittest.TestCase): 200, 201, 201, 201) # HEAD PUT PUT PUT self.assertEqual(seen_headers, [ {'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003', - 'X-Account-Partition': 1, + 'X-Account-Partition': '1', 'X-Account-Device': 'sda,sdd'}, {'X-Account-Host': '10.0.0.1:1001', - 'X-Account-Partition': 1, + 'X-Account-Partition': '1', 'X-Account-Device': 'sdb'}, {'X-Account-Host': '10.0.0.2:1002', - 'X-Account-Partition': 1, + 'X-Account-Partition': '1', 'X-Account-Device': 'sdc'}]) def test_DELETE_x_account_headers_with_fewer_account_replicas(self): @@ -5227,10 +5237,10 @@ class TestContainerController(unittest.TestCase): 200, 204, 204, 204) # HEAD DELETE DELETE DELETE self.assertEqual(seen_headers, [ {'X-Account-Host': '10.0.0.0:1000', - 'X-Account-Partition': 1, + 'X-Account-Partition': '1', 'X-Account-Device': 'sda'}, {'X-Account-Host': '10.0.0.1:1001', - 'X-Account-Partition': 1, + 'X-Account-Partition': '1', 'X-Account-Device': 'sdb'}, {'X-Account-Host': None, 'X-Account-Partition': None, @@ -5246,13 +5256,13 @@ class TestContainerController(unittest.TestCase): 200, 204, 204, 204) # HEAD DELETE DELETE DELETE self.assertEqual(seen_headers, [ {'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003', - 'X-Account-Partition': 1, + 'X-Account-Partition': '1', 'X-Account-Device': 'sda,sdd'}, {'X-Account-Host': '10.0.0.1:1001', - 'X-Account-Partition': 1, + 'X-Account-Partition': '1', 'X-Account-Device': 'sdb'}, {'X-Account-Host': '10.0.0.2:1002', - 'X-Account-Partition': 1, + 'X-Account-Partition': '1', 'X-Account-Device': 'sdc'}])