From 6c50e9b3a1574b5b51ae39639637aee279499d6c Mon Sep 17 00:00:00 2001 From: gholt Date: Tue, 7 Jun 2011 23:19:48 +0000 Subject: [PATCH 1/6] You can specify X-Newest: true on GETs and HEADs to indicate you want Swift to query all backend copies and return the newest version retrieved. --- swift/obj/server.py | 2 ++ swift/proxy/server.py | 52 ++++++++++++++++++++++-------- test/unit/proxy/test_server.py | 59 +++++++++++++++++++++++++++++++--- 3 files changed, 95 insertions(+), 18 deletions(-) diff --git a/swift/obj/server.py b/swift/obj/server.py index 75766d112e..50c88a3c42 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -623,6 +623,7 @@ class ObjectController(object): file.keep_cache = True if 'Content-Encoding' in file.metadata: response.content_encoding = file.metadata['Content-Encoding'] + response.headers['X-Timestamp'] = file.metadata['X-Timestamp'] return request.get_response(response) def HEAD(self, request): @@ -657,6 +658,7 @@ class ObjectController(object): response.content_length = file_size if 'Content-Encoding' in file.metadata: response.content_encoding = file.metadata['Content-Encoding'] + response.headers['X-Timestamp'] = file.metadata['X-Timestamp'] return response def DELETE(self, request): diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 3300e7a384..de5f1cecae 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -41,8 +41,8 @@ from webob.exc import HTTPBadRequest, HTTPMethodNotAllowed, \ from webob import Request, Response from swift.common.ring import Ring -from swift.common.utils import get_logger, normalize_timestamp, split_path, \ - cache_from_env, ContextPool +from swift.common.utils import cache_from_env, ContextPool, get_logger, \ + normalize_timestamp, split_path, TRUE_VALUES from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ check_utf8, CONTAINER_LISTING_LIMIT, MAX_ACCOUNT_NAME_LENGTH, \ @@ -162,6 +162,7 @@ class SegmentedIterable(object): if self.segment > 10: sleep(max(self.next_get_time - time.time(), 0)) self.next_get_time = time.time() + 1 + shuffle(nodes) resp = self.controller.GETorHEAD_base(req, _('Object'), partition, self.controller.iter_nodes(partition, nodes, self.controller.app.object_ring), path, @@ -594,6 +595,8 @@ class Controller(object): statuses = [] reasons = [] bodies = [] + source = None + newest = req.headers.get('x-newest', 'f').lower() in TRUE_VALUES for node in nodes: if len(statuses) >= attempts: break @@ -606,23 +609,48 @@ class Controller(object): headers=req.headers, query_string=req.query_string) with Timeout(self.app.node_timeout): - source = conn.getresponse() + possible_source = conn.getresponse() except (Exception, TimeoutError): self.exception_occurred(node, server_type, _('Trying to %(method)s %(path)s') % {'method': req.method, 'path': req.path}) continue - if source.status == 507: + if possible_source.status == 507: self.error_limit(node) continue - if 200 <= source.status <= 399: + if 200 <= possible_source.status <= 399: # 404 if we know we don't have a synced copy - if not float(source.getheader('X-PUT-Timestamp', '1')): + if not float(possible_source.getheader('X-PUT-Timestamp', 1)): statuses.append(404) reasons.append('') bodies.append('') - source.read() + possible_source.read() continue + if (req.method == 'GET' and + possible_source.status in (200, 206)) or \ + 200 <= possible_source.status <= 399: + if newest: + ts = 0 + if source: + ts = float(source.getheader('x-put-timestamp', + source.getheader('x-timestamp', 0))) + pts = float(possible_source.getheader('x-put-timestamp', + possible_source.getheader('x-timestamp', 0))) + if pts > ts: + source = possible_source + continue + else: + source = possible_source + break + statuses.append(possible_source.status) + reasons.append(possible_source.reason) + bodies.append(possible_source.read()) + if possible_source.status >= 500: + self.error_occurred(node, _('ERROR %(status)d %(body)s ' \ + 'From %(type)s Server') % + {'status': possible_source.status, + 'body': bodies[-1][:1024], 'type': server_type}) + if source: if req.method == 'GET' and source.status in (200, 206): res = Response(request=req, conditional_response=True) res.bytes_transferred = 0 @@ -662,13 +690,6 @@ class Controller(object): res.charset = None res.content_type = source.getheader('Content-Type') return res - statuses.append(source.status) - reasons.append(source.reason) - bodies.append(source.read()) - if source.status >= 500: - self.error_occurred(node, _('ERROR %(status)d %(body)s ' \ - 'From %(type)s Server') % {'status': source.status, - 'body': bodies[-1][:1024], 'type': server_type}) return self.best_response(req, statuses, reasons, bodies, '%s %s' % (server_type, req.method)) @@ -723,6 +744,7 @@ class ObjectController(Controller): lreq = Request.blank('/%s/%s?prefix=%s&format=json&marker=%s' % (quote(self.account_name), quote(lcontainer), quote(lprefix), quote(marker))) + shuffle(lnodes) lresp = self.GETorHEAD_base(lreq, _('Container'), lpartition, lnodes, lreq.path_info, self.app.container_ring.replica_count) @@ -1174,6 +1196,7 @@ class ContainerController(Controller): return HTTPNotFound(request=req) part, nodes = self.app.container_ring.get_nodes( self.account_name, self.container_name) + shuffle(nodes) resp = self.GETorHEAD_base(req, _('Container'), part, nodes, req.path_info, self.app.container_ring.replica_count) @@ -1304,6 +1327,7 @@ class AccountController(Controller): def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" partition, nodes = self.app.account_ring.get_nodes(self.account_name) + shuffle(nodes) return self.GETorHEAD_base(req, _('Account'), partition, nodes, req.path_info.rstrip('/'), self.app.account_ring.replica_count) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index fe2d1ca01e..cbe8533f23 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -150,7 +150,7 @@ def fake_http_connect(*code_iter, **kwargs): class FakeConn(object): - def __init__(self, status, etag=None, body=''): + def __init__(self, status, etag=None, body='', timestamp='1'): self.status = status self.reason = 'Fake' self.host = '1.2.3.4' @@ -159,6 +159,7 @@ def fake_http_connect(*code_iter, **kwargs): self.received = 0 self.etag = etag self.body = body + self.timestamp = timestamp def getresponse(self): if kwargs.get('raise_exc'): @@ -173,7 +174,8 @@ def fake_http_connect(*code_iter, **kwargs): def getheaders(self): headers = {'content-length': len(self.body), 'content-type': 'x-application/test', - 'x-timestamp': '1', + 'x-timestamp': self.timestamp, + 'last-modified': self.timestamp, 'x-object-meta-test': 'testing', 'etag': self.etag or '"68b329da9893e34099c7d8ad5cb9c940"', @@ -209,7 +211,8 @@ def fake_http_connect(*code_iter, **kwargs): def getheader(self, name, default=None): return dict(self.getheaders()).get(name.lower(), default) - etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) + timestamps_iter = iter(kwargs.get('timestamps') or [None] * len(code_iter)) + etag_iter = iter(kwargs.get('etags') or ['1'] * len(code_iter)) x = kwargs.get('missing_container', [False] * len(code_iter)) if not isinstance(x, (tuple, list)): x = [x] * len(code_iter) @@ -226,9 +229,11 @@ def fake_http_connect(*code_iter, **kwargs): kwargs['give_connect'](*args, **ckwargs) status = code_iter.next() etag = etag_iter.next() + timestamp = timestamps_iter.next() if status == -1: raise HTTPException() - return FakeConn(status, etag, body=kwargs.get('body', '')) + return FakeConn(status, etag, body=kwargs.get('body', ''), + timestamp=timestamp) return connect @@ -986,6 +991,51 @@ class TestObjectController(unittest.TestCase): test_status_map((404, 404, 500), 404) test_status_map((500, 500, 500), 503) + def test_HEAD_newest(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + + def test_status_map(statuses, expected, timestamps, + expected_timestamp): + proxy_server.http_connect = \ + fake_http_connect(*statuses, timestamps=timestamps) + self.app.memcache.store = {} + req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'}) + self.app.update_request(req) + res = controller.HEAD(req) + self.assertEquals(res.status[:len(str(expected))], + str(expected)) + self.assertEquals(res.headers.get('last-modified'), + expected_timestamp) + + test_status_map((200, 200, 200), 200, ('1', '2', '3'), '3') + test_status_map((200, 200, 200), 200, ('1', '3', '2'), '3') + test_status_map((200, 200, 200), 200, ('1', '3', '1'), '3') + test_status_map((200, 200, 200), 200, ('3', '3', '1'), '3') + + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + + def test_status_map(statuses, expected, timestamps, + expected_timestamp): + proxy_server.http_connect = \ + fake_http_connect(*statuses, timestamps=timestamps) + self.app.memcache.store = {} + req = Request.blank('/a/c/o', {}) + self.app.update_request(req) + res = controller.HEAD(req) + self.assertEquals(res.status[:len(str(expected))], + str(expected)) + self.assertEquals(res.headers.get('last-modified'), + expected_timestamp) + + test_status_map((200, 200, 200), 200, ('1', '2', '3'), '1') + test_status_map((200, 200, 200), 200, ('1', '3', '2'), '1') + test_status_map((200, 200, 200), 200, ('1', '3', '1'), '1') + test_status_map((200, 200, 200), 200, ('3', '3', '1'), '3') + def test_POST_meta_val_len(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', @@ -2772,6 +2822,7 @@ class TestContainerController(unittest.TestCase): def test_error_limiting(self): with save_globals(): + proxy_server.shuffle = lambda l: None controller = proxy_server.ContainerController(self.app, 'account', 'container') self.assert_status_map(controller.HEAD, (200, 503, 200, 200), 200, From 37fbf5ab77ecb2a3758cd8889bf34e1384b8d240 Mon Sep 17 00:00:00 2001 From: gholt Date: Wed, 8 Jun 2011 02:26:16 +0000 Subject: [PATCH 2/6] Object COPY requests now always copy the newest object they can find. --- swift/proxy/server.py | 13 ++++-- test/unit/proxy/test_server.py | 72 ++++++++++++++++++++++++++++------ 2 files changed, 69 insertions(+), 16 deletions(-) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index de5f1cecae..b092f7f9a4 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -632,10 +632,10 @@ class Controller(object): if newest: ts = 0 if source: - ts = float(source.getheader('x-put-timestamp', - source.getheader('x-timestamp', 0))) - pts = float(possible_source.getheader('x-put-timestamp', - possible_source.getheader('x-timestamp', 0))) + ts = float(source.getheader('x-put-timestamp') or + source.getheader('x-timestamp') or 0) + pts = float(possible_source.getheader('x-put-timestamp') or + possible_source.getheader('x-timestamp') or 0) if pts > ts: source = possible_source continue @@ -956,6 +956,7 @@ class ObjectController(Controller): reader = req.environ['wsgi.input'].read data_source = iter(lambda: reader(self.app.client_chunk_size), '') source_header = req.headers.get('X-Copy-From') + source_resp = None if source_header: source_header = unquote(source_header) acct = req.path_info.split('/', 2)[1] @@ -971,6 +972,7 @@ class ObjectController(Controller): '/') source_req = req.copy_get() source_req.path_info = source_header + source_req.headers['X-Newest'] = 'true' orig_obj_name = self.object_name orig_container_name = self.container_name self.object_name = src_obj_name @@ -1103,6 +1105,9 @@ class ObjectController(Controller): if source_header: resp.headers['X-Copied-From'] = quote( source_header.split('/', 2)[2]) + if 'last-modified' in source_resp.headers: + resp.headers['X-Copied-From-Last-Modified'] = \ + source_resp.headers['last-modified'] for k, v in req.headers.items(): if k.lower().startswith('x-object-meta-'): resp.headers[k] = v diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index cbe8533f23..cc93877c45 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -211,8 +211,8 @@ def fake_http_connect(*code_iter, **kwargs): def getheader(self, name, default=None): return dict(self.getheaders()).get(name.lower(), default) - timestamps_iter = iter(kwargs.get('timestamps') or [None] * len(code_iter)) - etag_iter = iter(kwargs.get('etags') or ['1'] * len(code_iter)) + timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) + etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) x = kwargs.get('missing_container', [False] * len(code_iter)) if not isinstance(x, (tuple, list)): x = [x] * len(code_iter) @@ -1014,6 +1014,29 @@ class TestObjectController(unittest.TestCase): test_status_map((200, 200, 200), 200, ('1', '3', '1'), '3') test_status_map((200, 200, 200), 200, ('3', '3', '1'), '3') + def test_GET_newest(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + + def test_status_map(statuses, expected, timestamps, + expected_timestamp): + proxy_server.http_connect = \ + fake_http_connect(*statuses, timestamps=timestamps) + self.app.memcache.store = {} + req = Request.blank('/a/c/o', {}, headers={'x-newest': 'true'}) + self.app.update_request(req) + res = controller.GET(req) + self.assertEquals(res.status[:len(str(expected))], + str(expected)) + self.assertEquals(res.headers.get('last-modified'), + expected_timestamp) + + test_status_map((200, 200, 200), 200, ('1', '2', '3'), '3') + test_status_map((200, 200, 200), 200, ('1', '3', '2'), '3') + test_status_map((200, 200, 200), 200, ('1', '3', '1'), '3') + test_status_map((200, 200, 200), 200, ('3', '3', '1'), '3') + with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') @@ -1725,8 +1748,10 @@ class TestObjectController(unittest.TestCase): headers={'Destination': 'c/o'}) req.account = 'a' proxy_server.http_connect = \ - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc obj obj obj + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, + 201) + # acct cont acct cont objc objc objc obj obj + # obj self.app.memcache.store = {} resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) @@ -1738,8 +1763,10 @@ class TestObjectController(unittest.TestCase): req.account = 'a' controller.object_name = 'o/o2' proxy_server.http_connect = \ - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc obj obj obj + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, + 201) + # acct cont acct cont objc objc objc obj obj + # obj self.app.memcache.store = {} resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) @@ -1750,8 +1777,10 @@ class TestObjectController(unittest.TestCase): req.account = 'a' controller.object_name = 'o' proxy_server.http_connect = \ - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc obj obj obj + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, + 201) + # acct cont acct cont objc objc objc obj obj + # obj self.app.memcache.store = {} resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) @@ -1763,8 +1792,10 @@ class TestObjectController(unittest.TestCase): req.account = 'a' controller.object_name = 'o/o2' proxy_server.http_connect = \ - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc obj obj obj + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, + 201) + # acct cont acct cont objc objc objc obj obj + # obj self.app.memcache.store = {} resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) @@ -1820,8 +1851,8 @@ class TestObjectController(unittest.TestCase): req.account = 'a' controller.object_name = 'o' proxy_server.http_connect = \ - fake_http_connect(200, 200, 200, 201, 201, 201) - # acct cont objc obj obj obj + fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj self.app.memcache.store = {} resp = controller.COPY(req) self.assertEquals(resp.status_int, 201) @@ -1829,6 +1860,23 @@ class TestObjectController(unittest.TestCase): 'testing') self.assertEquals(resp.headers.get('x-object-meta-ours'), 'okay') + def test_COPY_newest(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o') + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'COPY'}, + headers={'Destination': '/c/o'}) + req.account = 'a' + controller.object_name = 'o' + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201, + timestamps=('1', '1', '1', '3', '2', '4', '4', '4')) + # acct cont objc objc objc obj obj obj + self.app.memcache.store = {} + resp = controller.COPY(req) + self.assertEquals(resp.status_int, 201) + self.assertEquals(resp.headers['x-copied-from-last-modified'], + '3') + def test_chunked_put(self): class ChunkedFile(): From 7c9e542c020e3ad846fa8b28385783e04daa48ce Mon Sep 17 00:00:00 2001 From: gholt Date: Wed, 8 Jun 2011 04:19:34 +0000 Subject: [PATCH 3/6] Implemented object POST as COPY --- doc/source/deployment_guide.rst | 10 ++ etc/proxy-server.conf-sample | 5 + swift/common/direct_client.py | 51 +++++++++ swift/proxy/server.py | 72 +++++++----- test/functional/swift.py | 2 +- test/functional/tests.py | 4 +- test/probe/test_account_failures.py | 8 +- test/probe/test_container_failures.py | 23 ++-- test/probe/test_object_handoff.py | 84 +++++++------- test/unit/proxy/test_server.py | 154 +++++++++++++++++++++++--- 10 files changed, 312 insertions(+), 101 deletions(-) diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 9a78c56960..ce011d3fb9 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -547,6 +547,16 @@ error_suppression_limit 10 Error count to consider a node error limited allow_account_management false Whether account PUTs and DELETEs are even callable +post_as_copy true Set post_as_copy = false to turn + on fast posts where only the + metadata changes are stored anew + and the original data file is + kept in place. This makes for + quicker posts; but since the + container metadata isn't updated + in this mode, features like + container sync won't be able to + sync posts. ============================ =============== ============================= [auth] diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 9129e1e0ba..a7fa155422 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -40,6 +40,11 @@ use = egg:swift#proxy # If set to 'true' any authorized user may create and delete accounts; if # 'false' no one, even authorized, can. # allow_account_management = false +# Set post_as_copy = false to turn on fast posts where only the metadata +# changes are stored anew and the original data file is kept in place. This +# makes for quicker posts; but since the container metadata isn't updated in +# this mode, features like container sync won't be able to sync posts. +# post_as_copy = true [filter:swauth] use = egg:swift#swauth diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py index d7fde1f221..165ed3ed8c 100644 --- a/swift/common/direct_client.py +++ b/swift/common/direct_client.py @@ -36,6 +36,57 @@ def quote(value, safe='/'): return _quote(value, safe) +def direct_get_account(node, part, account, marker=None, limit=None, + prefix=None, delimiter=None, conn_timeout=5, + response_timeout=15): + """ + Get listings directly from the account server. + + :param node: node dictionary from the ring + :param part: partition the account is on + :param account: account name + :param marker: marker query + :param limit: query limit + :param prefix: prefix query + :param delimeter: delimeter for the query + :param conn_timeout: timeout in seconds for establishing the connection + :param response_timeout: timeout in seconds for getting the response + :returns: a tuple of (response headers, a list of containers) The response + headers will be a dict and all header names will be lowercase. + """ + path = '/' + account + qs = 'format=json' + if marker: + qs += '&marker=%s' % quote(marker) + if limit: + qs += '&limit=%d' % limit + if prefix: + qs += '&prefix=%s' % quote(prefix) + if delimiter: + qs += '&delimiter=%s' % quote(delimiter) + with Timeout(conn_timeout): + conn = http_connect(node['ip'], node['port'], node['device'], part, + 'GET', path, query_string='format=json') + with Timeout(response_timeout): + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + resp.read() + raise ClientException( + 'Account server %s:%s direct GET %s gave status %s' % (node['ip'], + node['port'], repr('/%s/%s%s' % (node['device'], part, path)), + resp.status), + http_host=node['ip'], http_port=node['port'], + http_device=node['device'], http_status=resp.status, + http_reason=resp.reason) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + if resp.status == 204: + resp.read() + return resp_headers, [] + return resp_headers, json_loads(resp.read()) + + def direct_head_container(node, part, account, container, conn_timeout=5, response_timeout=15): """ diff --git a/swift/proxy/server.py b/swift/proxy/server.py index b092f7f9a4..0d0e0d944a 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -872,29 +872,39 @@ class ObjectController(Controller): @delay_denial def POST(self, req): """HTTP POST request handler.""" - error_response = check_metadata(req, 'object') - if error_response: - return error_response - container_partition, containers, _junk, req.acl = \ - self.container_info(self.account_name, self.container_name) - if 'swift.authorize' in req.environ: - aresp = req.environ['swift.authorize'](req) - if aresp: - return aresp - if not containers: - return HTTPNotFound(request=req) - partition, nodes = self.app.object_ring.get_nodes( - self.account_name, self.container_name, self.object_name) - req.headers['X-Timestamp'] = normalize_timestamp(time.time()) - headers = [] - for container in containers: - nheaders = dict(req.headers.iteritems()) - nheaders['X-Container-Host'] = '%(ip)s:%(port)s' % container - nheaders['X-Container-Partition'] = container_partition - nheaders['X-Container-Device'] = container['device'] - headers.append(nheaders) - return self.make_requests(req, self.app.object_ring, - partition, 'POST', req.path_info, headers) + if self.app.post_as_copy: + req.method = 'PUT' + req.path_info = '/%s/%s/%s' % (self.account_name, + self.container_name, self.object_name) + req.headers['Content-Length'] = 0 + req.headers['X-Copy-From'] = '/%s/%s' % (self.container_name, + self.object_name) + req.headers['X-Fresh-Metadata'] = 'true' + return self.PUT(req) + else: + error_response = check_metadata(req, 'object') + if error_response: + return error_response + container_partition, containers, _junk, req.acl = \ + self.container_info(self.account_name, self.container_name) + if 'swift.authorize' in req.environ: + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp + if not containers: + return HTTPNotFound(request=req) + partition, nodes = self.app.object_ring.get_nodes( + self.account_name, self.container_name, self.object_name) + req.headers['X-Timestamp'] = normalize_timestamp(time.time()) + headers = [] + for container in containers: + nheaders = dict(req.headers.iteritems()) + nheaders['X-Container-Host'] = '%(ip)s:%(port)s' % container + nheaders['X-Container-Partition'] = container_partition + nheaders['X-Container-Device'] = container['device'] + headers.append(nheaders) + return self.make_requests(req, self.app.object_ring, + partition, 'POST', req.path_info, headers) def _send_file(self, conn, path): """Method for a file PUT coro""" @@ -998,12 +1008,14 @@ class ObjectController(Controller): if not content_type_manually_set: new_req.headers['Content-Type'] = \ source_resp.headers['Content-Type'] - for k, v in source_resp.headers.items(): - if k.lower().startswith('x-object-meta-'): - new_req.headers[k] = v - for k, v in req.headers.items(): - if k.lower().startswith('x-object-meta-'): - new_req.headers[k] = v + if new_req.headers.get('x-fresh-metadata', 'false').lower() \ + not in TRUE_VALUES: + for k, v in source_resp.headers.items(): + if k.lower().startswith('x-object-meta-'): + new_req.headers[k] = v + for k, v in req.headers.items(): + if k.lower().startswith('x-object-meta-'): + new_req.headers[k] = v req = new_req node_iter = self.iter_nodes(partition, nodes, self.app.object_ring) pile = GreenPile(len(nodes)) @@ -1431,6 +1443,8 @@ class BaseApplication(object): int(conf.get('recheck_account_existence', 60)) self.allow_account_management = \ conf.get('allow_account_management', 'false').lower() == 'true' + self.post_as_copy = \ + conf.get('post_as_copy', 'true').lower() in TRUE_VALUES self.resellers_conf = ConfigParser() self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf')) self.object_ring = object_ring or \ diff --git a/test/functional/swift.py b/test/functional/swift.py index 9d395511b2..c9180e68da 100644 --- a/test/functional/swift.py +++ b/test/functional/swift.py @@ -668,7 +668,7 @@ class File(Base): self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) - if self.conn.response.status != 202: + if self.conn.response.status not in (201, 202): raise ResponseError(self.conn.response) return True diff --git a/test/functional/tests.py b/test/functional/tests.py index 59dcf38960..c24bd87cf0 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -1032,7 +1032,7 @@ class TestFile(Base): self.assert_(file.write()) self.assert_status(201) self.assert_(file.sync_metadata()) - self.assert_status(202) + self.assert_status((201, 202)) else: self.assertRaises(ResponseError, file.write) self.assert_status(400) @@ -1245,7 +1245,7 @@ class TestFile(Base): file.metadata = metadata self.assert_(file.sync_metadata()) - self.assert_status(202) + self.assert_status((201, 202)) file = self.env.container.file(file.name) self.assert_(file.initialize()) diff --git a/test/probe/test_account_failures.py b/test/probe/test_account_failures.py index 807e397a57..29bfb6fc6f 100755 --- a/test/probe/test_account_failures.py +++ b/test/probe/test_account_failures.py @@ -20,7 +20,7 @@ from signal import SIGTERM from subprocess import Popen from time import sleep -from swift.common import client +from swift.common import client, direct_client from test.probe.common import get_to_final_state, kill_pids, reset_environment @@ -146,7 +146,8 @@ class TestAccountFailures(unittest.TestCase): sleep(2) # This is the earlier counts and bytes because the first node doesn't # have the newest udpates yet. - headers, containers = client.get_account(self.url, self.token) + headers, containers = \ + direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') @@ -167,7 +168,8 @@ class TestAccountFailures(unittest.TestCase): self.assert_(found2) get_to_final_state() - headers, containers = client.get_account(self.url, self.token) + headers, containers = \ + direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-bytes-used'], '9') diff --git a/test/probe/test_container_failures.py b/test/probe/test_container_failures.py index a493bffc27..005ece6290 100755 --- a/test/probe/test_container_failures.py +++ b/test/probe/test_container_failures.py @@ -24,7 +24,7 @@ from uuid import uuid4 import eventlet import sqlite3 -from swift.common import client +from swift.common import client, direct_client from swift.common.utils import hash_path, readconf from test.probe.common import get_to_final_state, kill_pids, reset_environment @@ -72,7 +72,8 @@ class TestContainerFailures(unittest.TestCase): # This okay because the first node hasn't got the update that the # object was deleted yet. self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)[1]]) + direct_client.direct_get_container(cnodes[0], cpart, + self.account, container)[1]]) # Unfortunately, the following might pass or fail, depending on the # position of the account server associated with the first container @@ -88,7 +89,8 @@ class TestContainerFailures(unittest.TestCase): client.put_object(self.url, self.token, container, object2, 'test') # First node still doesn't know object1 was deleted yet; this is okay. self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)[1]]) + direct_client.direct_get_container(cnodes[0], cpart, + self.account, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) @@ -150,7 +152,8 @@ class TestContainerFailures(unittest.TestCase): # server has to indicate the container exists for the put to continue. client.put_object(self.url, self.token, container, object2, 'test') self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)[1]]) + direct_client.direct_get_container(cnodes[0], cpart, + self.account, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) @@ -201,7 +204,8 @@ class TestContainerFailures(unittest.TestCase): # This okay because the first node hasn't got the update that the # object was deleted yet. self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)[1]]) + direct_client.direct_get_container(cnodes[0], cpart, + self.account, container)[1]]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got @@ -228,7 +232,8 @@ class TestContainerFailures(unittest.TestCase): client.put_object(self.url, self.token, container, object2, 'test') # First node still doesn't know object1 was deleted yet; this is okay. self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)[1]]) + direct_client.direct_get_container(cnodes[0], cpart, + self.account, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) @@ -277,7 +282,8 @@ class TestContainerFailures(unittest.TestCase): self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)[1]]) + direct_client.direct_get_container(cnodes[0], cpart, + self.account, container)[1]]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got @@ -303,7 +309,8 @@ class TestContainerFailures(unittest.TestCase): # server has to indicate the container exists for the put to continue. client.put_object(self.url, self.token, container, object2, 'test') self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)[1]]) + direct_client.direct_get_container(cnodes[0], cpart, + self.account, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py index 212fcc2c5e..8a4f9986ab 100755 --- a/test/probe/test_object_handoff.py +++ b/test/probe/test_object_handoff.py @@ -124,47 +124,49 @@ class TestObjectHandoff(unittest.TestCase): if not exc: raise Exception('Handoff object server still had test object') - kill(self.pids[self.port2server[onode['port']]], SIGTERM) - client.post_object(self.url, self.token, container, obj, - headers={'x-object-meta-probe': 'value'}) - oheaders = client.head_object(self.url, self.token, container, obj) - if oheaders.get('x-object-meta-probe') != 'value': - raise Exception('Metadata incorrect, was %s' % repr(oheaders)) - exc = False - try: - direct_client.direct_get_object(another_onode, opart, self.account, - container, obj) - except Exception: - exc = True - if not exc: - raise Exception('Handoff server claimed it had the object when ' - 'it should not have it') - self.pids[self.port2server[onode['port']]] = Popen([ - 'swift-object-server', - '/etc/swift/object-server/%d.conf' % - ((onode['port'] - 6000) / 10)]).pid - sleep(2) - oheaders = direct_client.direct_get_object(onode, opart, self.account, - container, obj)[0] - if oheaders.get('x-object-meta-probe') == 'value': - raise Exception('Previously downed object server had the new ' - 'metadata when it should not have it') - # Run the extra server last so it'll remove it's extra partition - ps = [] - for n in onodes: - ps.append(Popen(['swift-object-replicator', - '/etc/swift/object-server/%d.conf' % - ((n['port'] - 6000) / 10), 'once'])) - for p in ps: - p.wait() - call(['swift-object-replicator', - '/etc/swift/object-server/%d.conf' % - ((another_onode['port'] - 6000) / 10), 'once']) - oheaders = direct_client.direct_get_object(onode, opart, self.account, - container, obj)[0] - if oheaders.get('x-object-meta-probe') != 'value': - raise Exception( - 'Previously downed object server did not have the new metadata') +# Because POST has changed to a COPY by default, POSTs will succeed on all up +# nodes now if at least one up node has the object. +# kill(self.pids[self.port2server[onode['port']]], SIGTERM) +# client.post_object(self.url, self.token, container, obj, +# headers={'x-object-meta-probe': 'value'}) +# oheaders = client.head_object(self.url, self.token, container, obj) +# if oheaders.get('x-object-meta-probe') != 'value': +# raise Exception('Metadata incorrect, was %s' % repr(oheaders)) +# exc = False +# try: +# direct_client.direct_get_object(another_onode, opart, self.account, +# container, obj) +# except Exception: +# exc = True +# if not exc: +# raise Exception('Handoff server claimed it had the object when ' +# 'it should not have it') +# self.pids[self.port2server[onode['port']]] = Popen([ +# 'swift-object-server', +# '/etc/swift/object-server/%d.conf' % +# ((onode['port'] - 6000) / 10)]).pid +# sleep(2) +# oheaders = direct_client.direct_get_object(onode, opart, self.account, +# container, obj)[0] +# if oheaders.get('x-object-meta-probe') == 'value': +# raise Exception('Previously downed object server had the new ' +# 'metadata when it should not have it') +# # Run the extra server last so it'll remove it's extra partition +# ps = [] +# for n in onodes: +# ps.append(Popen(['swift-object-replicator', +# '/etc/swift/object-server/%d.conf' % +# ((n['port'] - 6000) / 10), 'once'])) +# for p in ps: +# p.wait() +# call(['swift-object-replicator', +# '/etc/swift/object-server/%d.conf' % +# ((another_onode['port'] - 6000) / 10), 'once']) +# oheaders = direct_client.direct_get_object(onode, opart, self.account, +# container, obj)[0] +# if oheaders.get('x-object-meta-probe') != 'value': +# raise Exception( +# 'Previously downed object server did not have the new metadata') kill(self.pids[self.port2server[onode['port']]], SIGTERM) client.delete_object(self.url, self.token, container, obj) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index cc93877c45..0c9fa1aafd 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -925,6 +925,7 @@ class TestObjectController(unittest.TestCase): def test_POST(self): with save_globals(): + self.app.post_as_copy = False controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') @@ -945,6 +946,28 @@ class TestObjectController(unittest.TestCase): test_status_map((200, 200, 404, 500, 500), 503) test_status_map((200, 200, 404, 404, 404), 404) + def test_POST_as_copy(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + + def test_status_map(statuses, expected): + proxy_server.http_connect = fake_http_connect(*statuses) + self.app.memcache.store = {} + req = Request.blank('/a/c/o', {}, headers={ + 'Content-Type': 'foo/bar'}) + self.app.update_request(req) + res = controller.POST(req) + expected = str(expected) + self.assertEquals(res.status[:len(expected)], expected) + test_status_map((200, 200, 200, 200, 200, 202, 202, 202), 202) + test_status_map((200, 200, 200, 200, 200, 202, 202, 500), 202) + test_status_map((200, 200, 200, 200, 200, 202, 500, 500), 503) + test_status_map((200, 200, 200, 200, 200, 202, 404, 500), 503) + test_status_map((200, 200, 200, 200, 200, 202, 404, 404), 404) + test_status_map((200, 200, 200, 200, 200, 404, 500, 500), 503) + test_status_map((200, 200, 200, 200, 200, 404, 404, 404), 404) + def test_DELETE(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', @@ -1061,6 +1084,7 @@ class TestObjectController(unittest.TestCase): def test_POST_meta_val_len(self): with save_globals(): + self.app.post_as_copy = False controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = \ @@ -1080,8 +1104,30 @@ class TestObjectController(unittest.TestCase): res = controller.POST(req) self.assertEquals(res.status_int, 400) + def test_POST_as_copy_meta_val_len(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, 200, 200, 202, 202, 202) + # acct cont objc objc objc obj obj obj + req = Request.blank('/a/c/o', {}, headers={ + 'Content-Type': 'foo/bar', + 'X-Object-Meta-Foo': 'x' * 256}) + self.app.update_request(req) + res = controller.POST(req) + self.assertEquals(res.status_int, 202) + proxy_server.http_connect = fake_http_connect(202, 202, 202) + req = Request.blank('/a/c/o', {}, headers={ + 'Content-Type': 'foo/bar', + 'X-Object-Meta-Foo': 'x' * 257}) + self.app.update_request(req) + res = controller.POST(req) + self.assertEquals(res.status_int, 400) + def test_POST_meta_key_len(self): with save_globals(): + self.app.post_as_copy = False controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = \ @@ -1101,6 +1147,27 @@ class TestObjectController(unittest.TestCase): res = controller.POST(req) self.assertEquals(res.status_int, 400) + def test_POST_as_copy_meta_key_len(self): + with save_globals(): + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, 200, 200, 202, 202, 202) + # acct cont objc objc objc obj obj obj + req = Request.blank('/a/c/o', {}, headers={ + 'Content-Type': 'foo/bar', + ('X-Object-Meta-' + 'x' * 128): 'x'}) + self.app.update_request(req) + res = controller.POST(req) + self.assertEquals(res.status_int, 202) + proxy_server.http_connect = fake_http_connect(202, 202, 202) + req = Request.blank('/a/c/o', {}, headers={ + 'Content-Type': 'foo/bar', + ('X-Object-Meta-' + 'x' * 129): 'x'}) + self.app.update_request(req) + res = controller.POST(req) + self.assertEquals(res.status_int, 400) + def test_POST_meta_count(self): with save_globals(): controller = proxy_server.ObjectController(self.app, 'account', @@ -1375,7 +1442,8 @@ class TestObjectController(unittest.TestCase): self.assert_status_map(controller.HEAD, (200, 200, 200), 503) self.assert_('last_error' in controller.app.object_ring.devs[0]) self.assert_status_map(controller.PUT, (200, 201, 201, 201), 503) - self.assert_status_map(controller.POST, (200, 202, 202, 202), 503) + self.assert_status_map(controller.POST, + (200, 200, 200, 200, 202, 202, 202), 503) self.assert_status_map(controller.DELETE, (200, 204, 204, 204), 503) self.app.error_suppression_interval = -300 @@ -1468,18 +1536,41 @@ class TestObjectController(unittest.TestCase): def test_PUT_POST_requires_container_exist(self): with save_globals(): + self.app.post_as_copy = False self.app.memcache = FakeMemcacheReturnsNone() controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') + proxy_server.http_connect = \ - fake_http_connect(404, 404, 404, 200, 200, 200) + fake_http_connect(200, 404, 404, 404, 200, 200, 200) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 404) proxy_server.http_connect = \ - fake_http_connect(404, 404, 404, 200, 200, 200) + fake_http_connect(200, 404, 404, 404, 200, 200) + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'}, + headers={'Content-Type': 'text/plain'}) + self.app.update_request(req) + resp = controller.POST(req) + self.assertEquals(resp.status_int, 404) + + def test_PUT_POST_as_copy_requires_container_exist(self): + with save_globals(): + self.app.memcache = FakeMemcacheReturnsNone() + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + proxy_server.http_connect = \ + fake_http_connect(200, 404, 404, 404, 200, 200, 200) + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) + self.app.update_request(req) + resp = controller.PUT(req) + self.assertEquals(resp.status_int, 404) + + proxy_server.http_connect = \ + fake_http_connect(200, 404, 404, 404, 200, 200, 200, 200, 200, + 200) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'text/plain'}) self.app.update_request(req) @@ -1599,8 +1690,10 @@ class TestObjectController(unittest.TestCase): 'X-Copy-From': 'c/o'}) self.app.update_request(req) proxy_server.http_connect = \ - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc obj obj obj + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, + 201) + # acct cont acct cont objc objc objc obj obj + # obj self.app.memcache.store = {} resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) @@ -1612,8 +1705,8 @@ class TestObjectController(unittest.TestCase): 'X-Copy-From': 'c/o'}) self.app.update_request(req) proxy_server.http_connect = \ - fake_http_connect(200, 200, 200, 200, 200) - # acct cont acct cont objc + fake_http_connect(200, 200, 200, 200, 200, 200, 200) + # acct cont acct cont objc objc objc self.app.memcache.store = {} resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) @@ -1624,8 +1717,10 @@ class TestObjectController(unittest.TestCase): 'X-Copy-From': 'c/o/o2'}) req.account = 'a' proxy_server.http_connect = \ - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc obj obj obj + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, + 201) + # acct cont acct cont objc objc objc obj obj + # obj self.app.memcache.store = {} resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) @@ -1637,8 +1732,10 @@ class TestObjectController(unittest.TestCase): 'X-Copy-From': 'c/o%20o2'}) req.account = 'a' proxy_server.http_connect = \ - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc obj obj obj + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, + 201) + # acct cont acct cont objc objc objc obj obj + # obj self.app.memcache.store = {} resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) @@ -1650,8 +1747,10 @@ class TestObjectController(unittest.TestCase): 'X-Copy-From': '/c/o'}) self.app.update_request(req) proxy_server.http_connect = \ - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc obj obj obj + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, + 201) + # acct cont acct cont objc objc objc obj obj + # obj self.app.memcache.store = {} resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) @@ -1662,8 +1761,10 @@ class TestObjectController(unittest.TestCase): 'X-Copy-From': '/c/o/o2'}) req.account = 'a' proxy_server.http_connect = \ - fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) - # acct cont acct cont objc obj obj obj + fake_http_connect(200, 200, 200, 200, 200, 200, 200, 201, 201, + 201) + # acct cont acct cont objc objc objc obj obj + # obj self.app.memcache.store = {} resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) @@ -1723,8 +1824,8 @@ class TestObjectController(unittest.TestCase): 'X-Object-Meta-Ours': 'okay'}) self.app.update_request(req) proxy_server.http_connect = \ - fake_http_connect(200, 200, 200, 201, 201, 201) - # acct cont objc obj obj obj + fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) + # acct cont objc objc objc obj obj obj self.app.memcache.store = {} resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) @@ -2652,6 +2753,7 @@ class TestObjectController(unittest.TestCase): called[0] = True return HTTPUnauthorized(request=req) with save_globals(): + self.app.post_as_copy = False proxy_server.http_connect = \ fake_http_connect(200, 200, 201, 201, 201) controller = proxy_server.ObjectController(self.app, 'account', @@ -2663,6 +2765,24 @@ class TestObjectController(unittest.TestCase): res = controller.POST(req) self.assert_(called[0]) + def test_POST_as_copy_calls_authorize(self): + called = [False] + + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'}, + headers={'Content-Length': '5'}, body='12345') + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = controller.POST(req) + self.assert_(called[0]) + def test_PUT_calls_authorize(self): called = [False] From 41fcf63241f38c548c81877904e8651238faa8f5 Mon Sep 17 00:00:00 2001 From: gholt Date: Wed, 8 Jun 2011 04:29:24 +0000 Subject: [PATCH 4/6] Changed post_as_copy to object_post_as_copy --- doc/source/deployment_guide.rst | 10 +++++----- etc/proxy-server.conf-sample | 4 ++-- swift/proxy/server.py | 6 +++--- test/unit/proxy/test_server.py | 10 +++++----- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index ce011d3fb9..6086714028 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -547,11 +547,11 @@ error_suppression_limit 10 Error count to consider a node error limited allow_account_management false Whether account PUTs and DELETEs are even callable -post_as_copy true Set post_as_copy = false to turn - on fast posts where only the - metadata changes are stored anew - and the original data file is - kept in place. This makes for +object_post_as_copy true Set object_post_as_copy = false + to turn on fast posts where only + the metadata changes are stored + anew and the original data file + is kept in place. This makes for quicker posts; but since the container metadata isn't updated in this mode, features like diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index a7fa155422..c602b89dc9 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -40,11 +40,11 @@ use = egg:swift#proxy # If set to 'true' any authorized user may create and delete accounts; if # 'false' no one, even authorized, can. # allow_account_management = false -# Set post_as_copy = false to turn on fast posts where only the metadata +# Set object_post_as_copy = false to turn on fast posts where only the metadata # changes are stored anew and the original data file is kept in place. This # makes for quicker posts; but since the container metadata isn't updated in # this mode, features like container sync won't be able to sync posts. -# post_as_copy = true +# object_post_as_copy = true [filter:swauth] use = egg:swift#swauth diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 0d0e0d944a..343b08712b 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -872,7 +872,7 @@ class ObjectController(Controller): @delay_denial def POST(self, req): """HTTP POST request handler.""" - if self.app.post_as_copy: + if self.app.object_post_as_copy: req.method = 'PUT' req.path_info = '/%s/%s/%s' % (self.account_name, self.container_name, self.object_name) @@ -1443,8 +1443,8 @@ class BaseApplication(object): int(conf.get('recheck_account_existence', 60)) self.allow_account_management = \ conf.get('allow_account_management', 'false').lower() == 'true' - self.post_as_copy = \ - conf.get('post_as_copy', 'true').lower() in TRUE_VALUES + self.object_post_as_copy = \ + conf.get('object_post_as_copy', 'true').lower() in TRUE_VALUES self.resellers_conf = ConfigParser() self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf')) self.object_ring = object_ring or \ diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 0c9fa1aafd..0ab38013c6 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -925,7 +925,7 @@ class TestObjectController(unittest.TestCase): def test_POST(self): with save_globals(): - self.app.post_as_copy = False + self.app.object_post_as_copy = False controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') @@ -1084,7 +1084,7 @@ class TestObjectController(unittest.TestCase): def test_POST_meta_val_len(self): with save_globals(): - self.app.post_as_copy = False + self.app.object_post_as_copy = False controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = \ @@ -1127,7 +1127,7 @@ class TestObjectController(unittest.TestCase): def test_POST_meta_key_len(self): with save_globals(): - self.app.post_as_copy = False + self.app.object_post_as_copy = False controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = \ @@ -1536,7 +1536,7 @@ class TestObjectController(unittest.TestCase): def test_PUT_POST_requires_container_exist(self): with save_globals(): - self.app.post_as_copy = False + self.app.object_post_as_copy = False self.app.memcache = FakeMemcacheReturnsNone() controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') @@ -2753,7 +2753,7 @@ class TestObjectController(unittest.TestCase): called[0] = True return HTTPUnauthorized(request=req) with save_globals(): - self.app.post_as_copy = False + self.app.object_post_as_copy = False proxy_server.http_connect = \ fake_http_connect(200, 200, 201, 201, 201) controller = proxy_server.ObjectController(self.app, 'account', From 105d199b4b86b137e932e126489ad75045d58054 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Thu, 16 Jun 2011 11:04:17 -0700 Subject: [PATCH 5/6] fixing ratelimit tests --- test/unit/common/middleware/test_ratelimit.py | 141 +++++++++++------- 1 file changed, 91 insertions(+), 50 deletions(-) diff --git a/test/unit/common/middleware/test_ratelimit.py b/test/unit/common/middleware/test_ratelimit.py index 257033dc11..7f45c744c6 100644 --- a/test/unit/common/middleware/test_ratelimit.py +++ b/test/unit/common/middleware/test_ratelimit.py @@ -15,6 +15,7 @@ import unittest import time +import eventlet from contextlib import contextmanager from threading import Thread from webob import Request @@ -30,6 +31,7 @@ class FakeMemcache(object): def __init__(self): self.store = {} self.error_on_incr = False + self.init_incr_return_neg = False def get(self, key): return self.store.get(key) @@ -41,6 +43,10 @@ class FakeMemcache(object): def incr(self, key, delta=1, timeout=0): if self.error_on_incr: raise MemcacheConnectionError('Memcache restarting') + if self.init_incr_return_neg: + # simulate initial hit, force reset of memcache + self.init_incr_return_neg = False + return -10000000 self.store[key] = int(self.store.setdefault(key, 0)) + int(delta) if self.store[key] < 0: self.store[key] = 0 @@ -109,23 +115,53 @@ def dummy_filter_factory(global_conf, **local_conf): return ratelimit.RateLimitMiddleware(app, conf, logger=FakeLogger()) return limit_filter +time_ticker = 0 +time_override = [] + + +def mock_sleep(x): + global time_ticker + time_ticker += x + + +def mock_time(): + global time_override + global time_ticker + if time_override: + cur_time = time_override.pop(0) + if cur_time is None: + time_override = [None if i is None else i + time_ticker + for i in time_override] + return time_ticker + return cur_time + return time_ticker + class TestRateLimit(unittest.TestCase): - def _run(self, callable_func, num, rate, extra_sleep=0, - total_time=None, check_time=True): + def setUp(self): + global time_ticker + time_ticker = 0 + self.was_sleep = eventlet.sleep + eventlet.sleep = mock_sleep + self.was_time = time.time + time.time = mock_time + + def tearDown(self): + eventlet.sleep = self.was_sleep + time.time = self.was_time + + def _run(self, callable_func, num, rate, check_time=True): + global time_ticker begin = time.time() for x in range(0, num): result = callable_func() - # Extra sleep is here to test with different call intervals. - time.sleep(extra_sleep) end = time.time() - if total_time is None: - total_time = num / rate + total_time = float(num) / rate - 1.0 / rate # 1st request isn't limited # Allow for one second of variation in the total time. time_diff = abs(total_time - (end - begin)) if check_time: - self.assertTrue(time_diff < 1) + self.assertEquals(round(total_time, 1), round(time_ticker, 1)) return time_diff def test_get_container_maxrate(self): @@ -163,8 +199,8 @@ class TestRateLimit(unittest.TestCase): 'PUT', 'a', 'c', 'o')), 1) def test_ratelimit(self): - current_rate = 13 - num_calls = 5 + current_rate = 5 + num_calls = 50 conf_dict = {'account_ratelimit': current_rate} self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) @@ -172,9 +208,27 @@ class TestRateLimit(unittest.TestCase): req.environ['swift.cache'] = FakeMemcache() make_app_call = lambda: self.test_ratelimit(req.environ, start_response) + begin = time.time() self._run(make_app_call, num_calls, current_rate) + self.assertEquals(round(time.time() - begin, 1), 9.8) + + def test_ratelimit_set_incr(self): + current_rate = 5 + num_calls = 50 + conf_dict = {'account_ratelimit': current_rate} + self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) + ratelimit.http_connect = mock_http_connect(204) + req = Request.blank('/v/a') + req.environ['swift.cache'] = FakeMemcache() + req.environ['swift.cache'].init_incr_return_neg = True + make_app_call = lambda: self.test_ratelimit(req.environ, + start_response) + begin = time.time() + self._run(make_app_call, num_calls, current_rate, check_time=False) + self.assertEquals(round(time.time() - begin, 1), 9.8) def test_ratelimit_whitelist(self): + global time_ticker current_rate = 2 conf_dict = {'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2, @@ -195,7 +249,6 @@ class TestRateLimit(unittest.TestCase): self.result = self.parent.test_ratelimit(req.environ, start_response) nt = 5 - begin = time.time() threads = [] for i in range(nt): rc = rate_caller(self) @@ -206,10 +259,10 @@ class TestRateLimit(unittest.TestCase): the_498s = [t for t in threads if \ ''.join(t.result).startswith('Slow down')] self.assertEquals(len(the_498s), 0) - time_took = time.time() - begin - self.assert_(time_took < 1) + self.assertEquals(time_ticker, 0) def test_ratelimit_blacklist(self): + global time_ticker current_rate = 2 conf_dict = {'account_ratelimit': current_rate, 'max_sleep_time_seconds': 2, @@ -231,7 +284,6 @@ class TestRateLimit(unittest.TestCase): self.result = self.parent.test_ratelimit(req.environ, start_response) nt = 5 - begin = time.time() threads = [] for i in range(nt): rc = rate_caller(self) @@ -242,49 +294,35 @@ class TestRateLimit(unittest.TestCase): the_497s = [t for t in threads if \ ''.join(t.result).startswith('Your account')] self.assertEquals(len(the_497s), 5) - time_took = time.time() - begin - self.assert_(round(time_took, 1) == 0) + self.assertEquals(time_ticker, 0) def test_ratelimit_max_rate_double(self): + global time_ticker + global time_override current_rate = 2 conf_dict = {'account_ratelimit': current_rate, 'clock_accuracy': 100, 'max_sleep_time_seconds': 1} - # making clock less accurate for nosetests running slow self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) self.test_ratelimit.log_sleep_time_seconds = .00001 req = Request.blank('/v/a') req.environ['swift.cache'] = FakeMemcache() - begin = time.time() - class rate_caller(Thread): - - def __init__(self, parent, name): - Thread.__init__(self) - self.parent = parent - self.name = name - - def run(self): - self.result1 = self.parent.test_ratelimit(req.environ, - start_response) - time.sleep(.1) - self.result2 = self.parent.test_ratelimit(req.environ, - start_response) - nt = 3 - threads = [] - for i in range(nt): - rc = rate_caller(self, "thread %s" % i) - rc.start() - threads.append(rc) - for thread in threads: - thread.join() - all_results = [''.join(t.result1) for t in threads] - all_results += [''.join(t.result2) for t in threads] - the_498s = [t for t in all_results if t.startswith('Slow down')] - self.assertEquals(len(the_498s), 2) - time_took = time.time() - begin - self.assert_(1.5 <= round(time_took, 1) < 1.7, time_took) + time_override = [0, 0, 0, 0, None] + # simulates 4 requests coming in at same time, then sleeping + r = self.test_ratelimit(req.environ, start_response) + mock_sleep(.1) + r = self.test_ratelimit(req.environ, start_response) + mock_sleep(.1) + r = self.test_ratelimit(req.environ, start_response) + self.assertEquals(r[0], 'Slow down') + mock_sleep(.1) + r = self.test_ratelimit(req.environ, start_response) + self.assertEquals(r[0], 'Slow down') + mock_sleep(.1) + r = self.test_ratelimit(req.environ, start_response) + self.assertEquals(r[0], '204 No Content') def test_ratelimit_max_rate_multiple_acc(self): num_calls = 4 @@ -319,9 +357,10 @@ class TestRateLimit(unittest.TestCase): threads.append(rc) for thread in threads: thread.join() + time_took = time.time() - begin - # the all 15 threads still take 1.5 secs - self.assert_(1.5 <= round(time_took, 1) < 1.7) +# the all 15 threads still take 1.5 secs + self.assertEquals(1.5, round(time_took, 1)) def test_ratelimit_acc_vrs_container(self): conf_dict = {'clock_accuracy': 1000, @@ -354,14 +393,13 @@ class TestRateLimit(unittest.TestCase): threads.append(rc) for thread in threads: thread.join() - begin = time.time() req.environ['swift.cache'].set(cont_key, {'container_size': 20}) begin = time.time() threads = [] runthreads(threads, 3) time_took = time.time() - begin - self.assert_(round(time_took, 1) == .4) + self.assertEquals(round(time_took, 1), .4) def test_call_invalid_path(self): env = {'REQUEST_METHOD': 'GET', @@ -393,7 +431,10 @@ class TestRateLimit(unittest.TestCase): req.environ['swift.cache'] = None make_app_call = lambda: self.test_ratelimit(req.environ, start_response) - self._run(make_app_call, num_calls, current_rate) + begin = time.time() + self._run(make_app_call, num_calls, current_rate, check_time=False) + time_took = time.time() - begin + self.assertEquals(round(time_took, 1), 0) # no memcache, no limiting def test_restarting_memcache(self): current_rate = 2 @@ -409,7 +450,7 @@ class TestRateLimit(unittest.TestCase): begin = time.time() self._run(make_app_call, num_calls, current_rate, check_time=False) time_took = time.time() - begin - self.assert_(round(time_took, 1) == 0) # no memcache, no limiting + self.assertEquals(round(time_took, 1), 0) # no memcache, no limiting if __name__ == '__main__': unittest.main() From 8062d32c19c6b0f2a63d168ffb007f1b48592b07 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Thu, 16 Jun 2011 11:09:15 -0700 Subject: [PATCH 6/6] fixing ratelimit tests --- test/unit/common/middleware/test_ratelimit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/common/middleware/test_ratelimit.py b/test/unit/common/middleware/test_ratelimit.py index 7f45c744c6..97d9c92525 100644 --- a/test/unit/common/middleware/test_ratelimit.py +++ b/test/unit/common/middleware/test_ratelimit.py @@ -359,7 +359,6 @@ class TestRateLimit(unittest.TestCase): thread.join() time_took = time.time() - begin -# the all 15 threads still take 1.5 secs self.assertEquals(1.5, round(time_took, 1)) def test_ratelimit_acc_vrs_container(self):