From 4d3c76ab851148e85be8cd5f48120eabd957ecab Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Mon, 9 May 2011 15:21:34 -0500 Subject: [PATCH] moved transaction id to catch errors middleware --- swift/account/server.py | 6 ++-- swift/common/middleware/catch_errors.py | 13 ++++++- swift/common/middleware/swauth.py | 2 +- swift/container/server.py | 6 ++-- swift/obj/server.py | 8 ++--- swift/proxy/server.py | 30 ++++++++-------- test/unit/common/middleware/test_except.py | 23 ++++++++++++ test/unit/proxy/test_server.py | 42 +--------------------- 8 files changed, 61 insertions(+), 69 deletions(-) diff --git a/swift/account/server.py b/swift/account/server.py index 54f13177f5..43c9ade2be 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -86,7 +86,7 @@ class AccountController(object): return Response(status='507 %s is not mounted' % drive) broker = self._get_account_broker(drive, part, account) if container: # put account container - if 'x-cf-trans-id' in req.headers: + if 'x-trans-id' in req.headers: broker.pending_timeout = 3 if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): @@ -296,7 +296,7 @@ class AccountController(object): def __call__(self, env, start_response): start_time = time.time() req = Request(env) - self.logger.txn_id = req.headers.get('x-cf-trans-id', None) + self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: @@ -319,7 +319,7 @@ class AccountController(object): time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', - req.headers.get('x-cf-trans-id', '-'), + req.headers.get('x-trans-id', '-'), req.referer or '-', req.user_agent or '-', trans_time, additional_info) diff --git a/swift/common/middleware/catch_errors.py b/swift/common/middleware/catch_errors.py index 716bda4da1..8f6802d2a1 100644 --- a/swift/common/middleware/catch_errors.py +++ b/swift/common/middleware/catch_errors.py @@ -15,6 +15,7 @@ from webob import Request from webob.exc import HTTPServerError +import uuid from swift.common.utils import get_logger @@ -29,13 +30,23 @@ class CatchErrorMiddleware(object): self.logger = get_logger(conf, log_route='catch-errors') def __call__(self, env, start_response): + trans_id = env.get('HTTP_X_TRANS_ID') + if not trans_id: + trans_id = uuid.uuid4().hex + env['HTTP_X_TRANS_ID'] = 'tx' + trans_id try: - return self.app(env, start_response) + + def my_start_response(status, response_headers, exc_info=None): + trans_header = ('x-trans-id', trans_id) + response_headers.append(trans_header) + return start_response(status, response_headers, exc_info) + return self.app(env, my_start_response) except Exception, err: self.logger.exception(_('Error: %s'), err) resp = HTTPServerError(request=Request(env), body='An error occurred', content_type='text/plain') + resp.headers['x-trans-id'] = trans_id return resp(env, start_response) diff --git a/swift/common/middleware/swauth.py b/swift/common/middleware/swauth.py index 328799a19a..5f51fb40bd 100644 --- a/swift/common/middleware/swauth.py +++ b/swift/common/middleware/swauth.py @@ -1360,7 +1360,7 @@ class Swauth(object): getattr(req, 'bytes_transferred', 0) or '-', getattr(response, 'bytes_transferred', 0) or '-', req.headers.get('etag', '-'), - req.headers.get('x-cf-trans-id', '-'), logged_headers or '-', + req.headers.get('x-trans-id', '-'), logged_headers or '-', trans_time))) diff --git a/swift/container/server.py b/swift/container/server.py index 3ced13d61f..01173c852f 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -96,7 +96,7 @@ class ContainerController(object): 'x-delete-timestamp': info['delete_timestamp'], 'x-object-count': info['object_count'], 'x-bytes-used': info['bytes_used'], - 'x-cf-trans-id': req.headers.get('X-Cf-Trans-Id', '-')} + 'x-trans-id': req.headers.get('x-trans-id', '-')} if req.headers.get('x-account-override-deleted', 'no').lower() == \ 'yes': account_headers['x-account-override-deleted'] = 'yes' @@ -385,7 +385,7 @@ class ContainerController(object): def __call__(self, env, start_response): start_time = time.time() req = Request(env) - self.logger.txn_id = req.headers.get('x-cf-trans-id', None) + self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: @@ -405,7 +405,7 @@ class ContainerController(object): time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', - req.headers.get('x-cf-trans-id', '-'), + req.headers.get('x-trans-id', '-'), req.referer or '-', req.user_agent or '-', trans_time) if req.method.upper() == 'REPLICATE': diff --git a/swift/obj/server.py b/swift/obj/server.py index 6e67dc0ff2..75766d112e 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -549,7 +549,7 @@ class ObjectController(object): 'x-content-type': file.metadata['Content-Type'], 'x-timestamp': file.metadata['X-Timestamp'], 'x-etag': file.metadata['ETag'], - 'x-cf-trans-id': request.headers.get('x-cf-trans-id', '-')}, + 'x-trans-id': request.headers.get('x-trans-id', '-')}, device) resp = HTTPCreated(request=request, etag=etag) return resp @@ -686,7 +686,7 @@ class ObjectController(object): file.unlinkold(metadata['X-Timestamp']) self.container_update('DELETE', account, container, obj, request.headers, {'x-timestamp': metadata['X-Timestamp'], - 'x-cf-trans-id': request.headers.get('x-cf-trans-id', '-')}, + 'x-trans-id': request.headers.get('x-trans-id', '-')}, device) resp = response_class(request=request) return resp @@ -719,7 +719,7 @@ class ObjectController(object): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) - self.logger.txn_id = req.headers.get('x-cf-trans-id', None) + self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: @@ -740,7 +740,7 @@ class ObjectController(object): time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.referer or '-', - req.headers.get('x-cf-trans-id', '-'), + req.headers.get('x-trans-id', '-'), req.user_agent or '-', trans_time) if req.method == 'REPLICATE': diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 1c9c2a88f5..75753ac26a 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -358,7 +358,7 @@ class Controller(object): result_code = 0 attempts_left = self.app.account_ring.replica_count path = '/%s' % account - headers = {'x-cf-trans-id': self.trans_id} + headers = {'x-trans-id': self.trans_id} for node in self.iter_nodes(partition, nodes, self.app.account_ring): try: with ConnectionTimeout(self.app.conn_timeout): @@ -430,7 +430,7 @@ class Controller(object): write_acl = None container_size = None attempts_left = self.app.container_ring.replica_count - headers = {'x-cf-trans-id': self.trans_id} + headers = {'x-trans-id': self.trans_id} for node in self.iter_nodes(partition, nodes, self.app.container_ring): try: with ConnectionTimeout(self.app.conn_timeout): @@ -1227,7 +1227,7 @@ class ContainerController(Controller): headers = [] for account in accounts: nheaders = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-cf-trans-id': self.trans_id, + 'x-trans-id': self.trans_id, 'X-Account-Host': '%(ip)s:%(port)s' % account, 'X-Account-Partition': account_partition, 'X-Account-Device': account['device']} @@ -1255,7 +1255,7 @@ class ContainerController(Controller): container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-cf-trans-id': self.trans_id} + 'x-trans-id': self.trans_id} headers.update(value for value in req.headers.iteritems() if value[0].lower() in self.pass_through_headers or value[0].lower().startswith('x-container-meta-')) @@ -1278,7 +1278,7 @@ class ContainerController(Controller): headers = [] for account in accounts: headers.append({'X-Timestamp': normalize_timestamp(time.time()), - 'X-Cf-Trans-Id': self.trans_id, + 'X-Trans-Id': self.trans_id, 'X-Account-Host': '%(ip)s:%(port)s' % account, 'X-Account-Partition': account_partition, 'X-Account-Device': account['device']}) @@ -1323,7 +1323,7 @@ class AccountController(Controller): account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-cf-trans-id': self.trans_id} + 'x-trans-id': self.trans_id} headers.update(value for value in req.headers.iteritems() if value[0].lower().startswith('x-account-meta-')) if self.app.memcache: @@ -1340,7 +1340,7 @@ class AccountController(Controller): account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'X-CF-Trans-Id': self.trans_id} + 'X-Trans-Id': self.trans_id} headers.update(value for value in req.headers.iteritems() if value[0].lower().startswith('x-account-meta-')) if self.app.memcache: @@ -1357,7 +1357,7 @@ class AccountController(Controller): account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'X-CF-Trans-Id': self.trans_id} + 'X-Trans-Id': self.trans_id} if self.app.memcache: self.app.memcache.delete('account%s' % req.path_info.rstrip('/')) return self.make_requests(req, self.app.account_ring, @@ -1448,7 +1448,7 @@ class BaseApplication(object): if self.memcache is None: self.memcache = cache_from_env(env) req = self.update_request(Request(env)) - trans_id = req.headers.get('x-cf-trans-id') + trans_id = req.headers.get('x-trans-id') if 'eventlet.posthooks' in env: env['eventlet.posthooks'].append( (self.posthooklogger, (req,), {})) @@ -1460,8 +1460,8 @@ class BaseApplication(object): # differently than full transmissions. resp = self.handle_request(req) self.posthooklogger(env, req) - if trans_id: - resp.headers['x-trans-id'] = trans_id + #if trans_id: + # resp.headers['x-trans-id'] = trans_id return resp(env, start_response) except Exception: print "EXCEPTION IN __call__: %s: %s" % \ @@ -1476,8 +1476,6 @@ class BaseApplication(object): def update_request(self, req): req.bytes_transferred = '-' req.client_disconnect = False - if 'x-cf-trans-id' not in req.headers: - req.headers['x-cf-trans-id'] = 'tx' + str(uuid.uuid4()) if 'x-storage-token' in req.headers and \ 'x-auth-token' not in req.headers: req.headers['x-auth-token'] = req.headers['x-storage-token'] @@ -1501,8 +1499,8 @@ class BaseApplication(object): return HTTPPreconditionFailed(request=req, body='Bad URL') controller = controller(self, **path_parts) - controller.trans_id = req.headers.get('x-cf-trans-id', '-') - self.logger.txn_id = req.headers.get('x-cf-trans-id', None) + controller.trans_id = req.headers.get('x-trans-id', '-') + self.logger.txn_id = req.headers.get('x-trans-id', None) try: handler = getattr(controller, req.method) if not getattr(handler, 'publicly_accessible'): @@ -1582,7 +1580,7 @@ class Application(BaseApplication): getattr(req, 'bytes_transferred', 0) or '-', getattr(response, 'bytes_transferred', 0) or '-', req.headers.get('etag', '-'), - req.headers.get('x-cf-trans-id', '-'), + req.headers.get('x-trans-id', '-'), logged_headers or '-', trans_time, ))) diff --git a/test/unit/common/middleware/test_except.py b/test/unit/common/middleware/test_except.py index 89c6e9ea1f..b6e18960ba 100644 --- a/test/unit/common/middleware/test_except.py +++ b/test/unit/common/middleware/test_except.py @@ -45,5 +45,28 @@ class TestCatchErrors(unittest.TestCase): resp = app(req.environ, start_response) self.assertEquals(resp, ['An error occurred']) + def test_trans_id_header(self): + + def start_response(status, headers): + self.assert_('x-trans-id' in (x[0] for x in headers)) + app = catch_errors.CatchErrorMiddleware(FakeApp(), {}) + req = Request.blank('/v1/a') + app(req.environ, start_response) + app = catch_errors.CatchErrorMiddleware(FakeApp(), {}) + req = Request.blank('/v1/a/c') + app(req.environ, start_response) + app = catch_errors.CatchErrorMiddleware(FakeApp(), {}) + req = Request.blank('/v1/a/c/o') + app(req.environ, start_response) + app = catch_errors.CatchErrorMiddleware(FakeApp(True), {}) + req = Request.blank('/v1/a') + app(req.environ, start_response) + app = catch_errors.CatchErrorMiddleware(FakeApp(True), {}) + req = Request.blank('/v1/a/c') + app(req.environ, start_response) + app = catch_errors.CatchErrorMiddleware(FakeApp(True), {}) + req = Request.blank('/v1/a/c/o') + app(req.environ, start_response) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 272aaa7c4f..fe2d1ca01e 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -124,7 +124,7 @@ def setup(): for node in nodes: conn = proxy_server.http_connect(node['ip'], node['port'], node['device'], partition, 'PUT', '/a', - {'X-Timestamp': ts, 'X-CF-Trans-Id': 'test'}) + {'X-Timestamp': ts, 'x-trans-id': 'test'}) resp = conn.getresponse() assert(resp.status == 201) # Create container @@ -656,46 +656,6 @@ class TestProxyServer(unittest.TestCase): resp = app.handle_request(req) self.assert_(called[0]) - def test_trans_id_header(self): - - def start_response(status, headers): - self.assert_('x-trans-id' in (x[0] for x in headers)) - with save_globals(): - proxy_server.http_connect = fake_http_connect(200, 200, 200, 200) - app = proxy_server.Application(None, FakeMemcache(), - account_ring=FakeRing(), container_ring=FakeRing(), - object_ring=FakeRing()) - req = Request.blank('/v1/a') - app(req.environ, start_response) - - proxy_server.http_connect = fake_http_connect(200, 200, 200, 200) - app = proxy_server.Application(None, FakeMemcache(), - account_ring=FakeRing(), container_ring=FakeRing(), - object_ring=FakeRing()) - req = Request.blank('/v1/a/c') - app(req.environ, start_response) - - proxy_server.http_connect = fake_http_connect(200, 200, 200, 200) - app = proxy_server.Application(None, FakeMemcache(), - account_ring=FakeRing(), container_ring=FakeRing(), - object_ring=FakeRing()) - req = Request.blank('/v1/a/c/o') - app(req.environ, start_response) - - proxy_server.http_connect = fake_http_connect(200, 404, 404, 404) - app = proxy_server.Application(None, FakeMemcache(), - account_ring=FakeRing(), container_ring=FakeRing(), - object_ring=FakeRing()) - req = Request.blank('/v1/a/c/o') - app(req.environ, start_response) - - proxy_server.http_connect = fake_http_connect(200, 503, 503, 503) - app = proxy_server.Application(None, FakeMemcache(), - account_ring=FakeRing(), container_ring=FakeRing(), - object_ring=FakeRing()) - req = Request.blank('/v1/a/c/o') - app(req.environ, start_response) - class TestObjectController(unittest.TestCase):