From 97028e0b9c2523c9e35854957a8e49c544b1f0ba Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 4 Jan 2011 16:00:01 -0600 Subject: [PATCH 1/6] fixed missing NamedLogger import in bin/swift-bench, refactored LogAdapter.tnx_id so that it works with multiple calls to get_logger, fixed common.middleware.catch_errors to only call get_logger if it needs too, renamed x-cf-trans-id to x-swift-tnx-id --- bin/swift-bench | 8 ++-- swift/account/server.py | 6 +-- swift/common/middleware/catch_errors.py | 6 ++- swift/common/utils.py | 51 ++++++++++++++++++++----- swift/container/server.py | 6 +-- swift/obj/server.py | 8 ++-- swift/proxy/server.py | 24 ++++++------ test/unit/proxy/test_server.py | 2 +- 8 files changed, 74 insertions(+), 37 deletions(-) diff --git a/bin/swift-bench b/bin/swift-bench index ab332482cd..9c03dc3d3c 100755 --- a/bin/swift-bench +++ b/bin/swift-bench @@ -22,7 +22,7 @@ import uuid from optparse import OptionParser from swift.common.bench import BenchController -from swift.common.utils import readconf, NamedLogger +from swift.common.utils import readconf, LogAdapter, NamedFormatter # The defaults should be sufficient to run swift-bench on a SAIO CONF_DEFAULTS = { @@ -124,10 +124,10 @@ if __name__ == '__main__': 'critical': logging.CRITICAL}.get( options.log_level.lower(), logging.INFO)) loghandler = logging.StreamHandler() - logformat = logging.Formatter('%(asctime)s %(levelname)s %(message)s') - loghandler.setFormatter(logformat) logger.addHandler(loghandler) - logger = NamedLogger(logger, 'swift-bench') + logger = LogAdapter(logger) + logformat = NamedFormatter('swift-bench', logger, fmt='%(server)s %(asctime)s %(levelname)s %(message)s') + loghandler.setFormatter(logformat) controller = BenchController(logger, options) controller.run() diff --git a/swift/account/server.py b/swift/account/server.py index 53d604ce93..c782bdc951 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -86,7 +86,7 @@ class AccountController(object): return Response(status='507 %s is not mounted' % drive) broker = self._get_account_broker(drive, part, account) if container: # put account container - if 'x-cf-trans-id' in req.headers: + if 'x-swift-txn-id' in req.headers: broker.pending_timeout = 3 if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): @@ -296,7 +296,7 @@ class AccountController(object): def __call__(self, env, start_response): start_time = time.time() req = Request(env) - self.logger.txn_id = req.headers.get('x-cf-trans-id', None) + self.logger.txn_id = req.headers.get('x-swift-txn-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: @@ -319,7 +319,7 @@ class AccountController(object): time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', - req.headers.get('x-cf-trans-id', '-'), + req.headers.get('x-swift-txn-id', '-'), req.referer or '-', req.user_agent or '-', trans_time, additional_info) diff --git a/swift/common/middleware/catch_errors.py b/swift/common/middleware/catch_errors.py index 5fb8c33592..f1ab249b25 100644 --- a/swift/common/middleware/catch_errors.py +++ b/swift/common/middleware/catch_errors.py @@ -26,7 +26,11 @@ class CatchErrorMiddleware(object): def __init__(self, app, conf): self.app = app - self.logger = get_logger(conf) + try: + # if the application already has a logger we should use that one + self.logger = app.logger + except AttributeError: + self.logger = get_logger(conf) def __call__(self, env, start_response): try: diff --git a/swift/common/utils.py b/swift/common/utils.py index da71253e7b..78e8769bca 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -284,11 +284,15 @@ class LoggerFileObject(object): class LogAdapter(object): - """Cheesy version of the LoggerAdapter available in Python 3""" + """ + A Logger like object which performs some reformatting on calls to + :meth:`exception`. Can be used to store a threadlocal transaction id. + """ + + _txn_id = threading.local() def __init__(self, logger): self.logger = logger - self._txn_id = threading.local() for proxied_method in ('debug', 'log', 'warn', 'warning', 'error', 'critical', 'info'): setattr(self, proxied_method, getattr(logger, proxied_method)) @@ -334,19 +338,45 @@ class LogAdapter(object): class NamedFormatter(logging.Formatter): - def __init__(self, server, logger): - logging.Formatter.__init__(self) + """ + NamedFormatter is used to add additional information to log messages. + Normally it will simply add the server name as an attribute on the + LogRecord and the default format string will include it at the + begining of the log message. Additionally, if the transaction id is + available and not already included in the message, NamedFormatter will + add it. + + NamedFormatter may be initialized with a format string which makes use + of the standard LogRecord attributes. In addition the format string + may include the following mapping key: + + +----------------+---------------------------------------------+ + | Format | Description | + +================+=============================================+ + | %(server)s | Name of the swift server doing logging | + +----------------+---------------------------------------------+ + + :param server: the swift server name, a string. + :param logger: a Logger or :class:`LogAdapter` instance, additional + context may be pulled from attributes on this logger if + available. + :param fmt: the format string used to construct the message, if none is + supplied it defaults to ``"%(server)s %(message)s"`` + """ + + def __init__(self, server, logger, + fmt="%(server)s %(message)s"): + logging.Formatter.__init__(self, fmt) self.server = server self.logger = logger def format(self, record): + record.server = self.server msg = logging.Formatter.format(self, record) if self.logger.txn_id and (record.levelno != logging.INFO or self.logger.txn_id not in msg): - return '%s %s (txn: %s)' % (self.server, msg, self.logger.txn_id) - else: - return '%s %s' % (self.server, msg) - + msg = "%s (txn: %s)" % (msg, self.logger.txn_id) + return msg def get_logger(conf, name=None, log_to_console=False): """ @@ -386,7 +416,10 @@ def get_logger(conf, name=None, log_to_console=False): root_logger.setLevel( getattr(logging, conf.get('log_level', 'INFO').upper(), logging.INFO)) adapted_logger = LogAdapter(root_logger) - get_logger.handler.setFormatter(NamedFormatter(name, adapted_logger)) + formatter = NamedFormatter(name, adapted_logger) + get_logger.handler.setFormatter(formatter) + if hasattr(get_logger, 'console'): + get_logger.console.setFormatter(formatter) return adapted_logger diff --git a/swift/container/server.py b/swift/container/server.py index fc06194de6..58b09671ed 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -95,7 +95,7 @@ class ContainerController(object): 'x-delete-timestamp': info['delete_timestamp'], 'x-object-count': info['object_count'], 'x-bytes-used': info['bytes_used'], - 'x-cf-trans-id': req.headers.get('X-Cf-Trans-Id', '-')} + 'x-swift-txn-id': req.headers.get('x-swift-txn-id', '-')} if req.headers.get('x-account-override-deleted', 'no').lower() == \ 'yes': account_headers['x-account-override-deleted'] = 'yes' @@ -384,7 +384,7 @@ class ContainerController(object): def __call__(self, env, start_response): start_time = time.time() req = Request(env) - self.logger.txn_id = req.headers.get('x-cf-trans-id', None) + self.logger.txn_id = req.headers.get('x-swift-txn-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: @@ -404,7 +404,7 @@ class ContainerController(object): time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', - req.headers.get('x-cf-trans-id', '-'), + req.headers.get('x-swift-txn-id', '-'), req.referer or '-', req.user_agent or '-', trans_time) if req.method.upper() == 'REPLICATE': diff --git a/swift/obj/server.py b/swift/obj/server.py index 7c139d7775..280ddbf49a 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -409,7 +409,7 @@ class ObjectController(object): 'x-content-type': file.metadata['Content-Type'], 'x-timestamp': file.metadata['X-Timestamp'], 'x-etag': file.metadata['ETag'], - 'x-cf-trans-id': request.headers.get('x-cf-trans-id', '-')}, + 'x-swift-txn-id': request.headers.get('x-swift-txn-id', '-')}, device) resp = HTTPCreated(request=request, etag=etag) return resp @@ -531,7 +531,7 @@ class ObjectController(object): file.unlinkold(metadata['X-Timestamp']) self.container_update('DELETE', account, container, obj, request.headers, {'x-timestamp': metadata['X-Timestamp'], - 'x-cf-trans-id': request.headers.get('x-cf-trans-id', '-')}, + 'x-swift-txn-id': request.headers.get('x-swift-txn-id', '-')}, device) resp = response_class(request=request) return resp @@ -562,7 +562,7 @@ class ObjectController(object): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) - self.logger.txn_id = req.headers.get('x-cf-trans-id', None) + self.logger.txn_id = req.headers.get('x-swift-txn-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: @@ -583,7 +583,7 @@ class ObjectController(object): time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.referer or '-', - req.headers.get('x-cf-trans-id', '-'), + req.headers.get('x-swift-txn-id', '-'), req.user_agent or '-', trans_time) if req.method == 'REPLICATE': diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 1cde24cfe6..04187cc82b 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -356,7 +356,7 @@ class Controller(object): result_code = 0 attempts_left = self.app.account_ring.replica_count path = '/%s' % account - headers = {'x-cf-trans-id': self.trans_id} + headers = {'x-swift-txn-id': self.trans_id} for node in self.iter_nodes(partition, nodes, self.app.account_ring): if self.error_limited(node): continue @@ -430,7 +430,7 @@ class Controller(object): write_acl = None container_size = None attempts_left = self.app.container_ring.replica_count - headers = {'x-cf-trans-id': self.trans_id} + headers = {'x-swift-txn-id': self.trans_id} for node in self.iter_nodes(partition, nodes, self.app.container_ring): if self.error_limited(node): continue @@ -1247,7 +1247,7 @@ class ContainerController(Controller): container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-cf-trans-id': self.trans_id} + 'x-swift-txn-id': self.trans_id} headers.update(value for value in req.headers.iteritems() if value[0].lower() in self.pass_through_headers or value[0].lower().startswith('x-container-meta-')) @@ -1309,7 +1309,7 @@ class ContainerController(Controller): container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-cf-trans-id': self.trans_id} + 'x-swift-txn-id': self.trans_id} headers.update(value for value in req.headers.iteritems() if value[0].lower() in self.pass_through_headers or value[0].lower().startswith('x-container-meta-')) @@ -1362,7 +1362,7 @@ class ContainerController(Controller): container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-cf-trans-id': self.trans_id} + 'x-swift-txn-id': self.trans_id} statuses = [] reasons = [] bodies = [] @@ -1450,7 +1450,7 @@ class AccountController(Controller): account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-cf-trans-id': self.trans_id} + 'x-swift-txn-id': self.trans_id} headers.update(value for value in req.headers.iteritems() if value[0].lower().startswith('x-account-meta-')) statuses = [] @@ -1499,7 +1499,7 @@ class AccountController(Controller): account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'X-CF-Trans-Id': self.trans_id} + 'x-swift-txn-id': self.trans_id} headers.update(value for value in req.headers.iteritems() if value[0].lower().startswith('x-account-meta-')) statuses = [] @@ -1546,7 +1546,7 @@ class AccountController(Controller): account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'X-CF-Trans-Id': self.trans_id} + 'x-swift-txn-id': self.trans_id} statuses = [] reasons = [] bodies = [] @@ -1683,7 +1683,7 @@ class BaseApplication(object): def update_request(self, req): req.bytes_transferred = '-' req.client_disconnect = False - req.headers['x-cf-trans-id'] = 'tx' + str(uuid.uuid4()) + req.headers['x-swift-txn-id'] = 'tx' + str(uuid.uuid4()) if 'x-storage-token' in req.headers and \ 'x-auth-token' not in req.headers: req.headers['x-auth-token'] = req.headers['x-storage-token'] @@ -1707,8 +1707,8 @@ class BaseApplication(object): return HTTPPreconditionFailed(request=req, body='Bad URL') controller = controller(self, **path_parts) - controller.trans_id = req.headers.get('x-cf-trans-id', '-') - self.logger.txn_id = req.headers.get('x-cf-trans-id', None) + controller.trans_id = req.headers.get('x-swift-txn-id', '-') + self.logger.txn_id = req.headers.get('x-swift-txn-id', None) try: handler = getattr(controller, req.method) if not getattr(handler, 'publicly_accessible'): @@ -1786,7 +1786,7 @@ class Application(BaseApplication): getattr(req, 'bytes_transferred', 0) or '-', getattr(response, 'bytes_transferred', 0) or '-', req.headers.get('etag', '-'), - req.headers.get('x-cf-trans-id', '-'), + req.headers.get('x-swift-txn-id', '-'), logged_headers or '-', trans_time, ))) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 4577cd4dac..42d6a52e4d 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -1715,7 +1715,7 @@ class TestObjectController(unittest.TestCase): for node in nodes: conn = proxy_server.http_connect(node['ip'], node['port'], node['device'], partition, 'PUT', '/a', - {'X-Timestamp': ts, 'X-CF-Trans-Id': 'test'}) + {'X-Timestamp': ts, 'x-swift-txn-id': 'test'}) resp = conn.getresponse() self.assertEquals(resp.status, 201) # Head account, just a double check and really is here to test From 68a9acf9b8e85c8465c883ffd9ac9cbdbde5b7f5 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 4 Jan 2011 16:12:56 -0600 Subject: [PATCH 2/6] pep8 --- bin/swift-bench | 3 ++- swift/common/utils.py | 1 + test/unit/proxy/test_server.py | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/swift-bench b/bin/swift-bench index 9c03dc3d3c..4a6158614e 100755 --- a/bin/swift-bench +++ b/bin/swift-bench @@ -126,7 +126,8 @@ if __name__ == '__main__': loghandler = logging.StreamHandler() logger.addHandler(loghandler) logger = LogAdapter(logger) - logformat = NamedFormatter('swift-bench', logger, fmt='%(server)s %(asctime)s %(levelname)s %(message)s') + logformat = NamedFormatter('swift-bench', logger, + fmt='%(server)s %(asctime)s %(levelname)s %(message)s') loghandler.setFormatter(logformat) controller = BenchController(logger, options) diff --git a/swift/common/utils.py b/swift/common/utils.py index 78e8769bca..c240b242cf 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -378,6 +378,7 @@ class NamedFormatter(logging.Formatter): msg = "%s (txn: %s)" % (msg, self.logger.txn_id) return msg + def get_logger(conf, name=None, log_to_console=False): """ Get the current system logger using config settings. diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 42d6a52e4d..f7e2479a16 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -218,6 +218,7 @@ def save_globals(): # tests + class TestController(unittest.TestCase): def setUp(self): @@ -372,6 +373,7 @@ class TestController(unittest.TestCase): test(404, 507, 503) test(503, 503, 503) + class TestProxyServer(unittest.TestCase): def test_unhandled_exception(self): @@ -468,6 +470,7 @@ class TestObjectController(unittest.TestCase): 'text/html', 'text/html'])) test_content_type('test.css', iter(['', '', '', 'text/css', 'text/css', 'text/css'])) + def test_custom_mime_types_files(self): swift_dir = mkdtemp() try: From 6c1bf02a620c6cb9ccc3a72fa4b4154b6731147e Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 5 Jan 2011 09:32:19 -0600 Subject: [PATCH 3/6] pep8 fo' realz --- bin/swift-bench | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/swift-bench b/bin/swift-bench index 4a6158614e..9cdeea340e 100755 --- a/bin/swift-bench +++ b/bin/swift-bench @@ -127,7 +127,7 @@ if __name__ == '__main__': logger.addHandler(loghandler) logger = LogAdapter(logger) logformat = NamedFormatter('swift-bench', logger, - fmt='%(server)s %(asctime)s %(levelname)s %(message)s') + fmt='%(server)s %(asctime)s %(levelname)s %(message)s') loghandler.setFormatter(logformat) controller = BenchController(logger, options) From 25bf6ebfc3b979a54a2f731a5d1dfd5abb51b91b Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 5 Jan 2011 11:15:21 -0600 Subject: [PATCH 4/6] moved the txn_id read into a utils helper --- swift/account/server.py | 8 ++++---- swift/common/utils.py | 10 ++++++++++ swift/container/server.py | 8 ++++---- swift/obj/server.py | 10 +++++----- swift/proxy/server.py | 11 ++++++----- test/unit/common/test_utils.py | 14 ++++++++++++++ 6 files changed, 43 insertions(+), 18 deletions(-) diff --git a/swift/account/server.py b/swift/account/server.py index c782bdc951..ec9ad343dc 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -29,7 +29,7 @@ import simplejson from swift.common.db import AccountBroker from swift.common.utils import get_logger, get_param, hash_path, \ - normalize_timestamp, split_path, storage_directory + normalize_timestamp, split_path, storage_directory, get_txn_id from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.db_replicator import ReplicatorRpc @@ -86,7 +86,7 @@ class AccountController(object): return Response(status='507 %s is not mounted' % drive) broker = self._get_account_broker(drive, part, account) if container: # put account container - if 'x-swift-txn-id' in req.headers: + if get_txn_id(req, None) is None: broker.pending_timeout = 3 if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): @@ -296,7 +296,7 @@ class AccountController(object): def __call__(self, env, start_response): start_time = time.time() req = Request(env) - self.logger.txn_id = req.headers.get('x-swift-txn-id', None) + self.logger.txn_id = get_txn_id(req, None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: @@ -319,7 +319,7 @@ class AccountController(object): time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', - req.headers.get('x-swift-txn-id', '-'), + get_txn_id(req, '-'), req.referer or '-', req.user_agent or '-', trans_time, additional_info) diff --git a/swift/common/utils.py b/swift/common/utils.py index c240b242cf..d7abac2ec8 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -107,6 +107,16 @@ def get_param(req, name, default=None): value.decode('utf8') # Ensure UTF8ness return value +def get_txn_id(req, default=None): + """ + Get the transaction id from a request + + :param req: Webob request object + :param default: value to return if no transaction id is found + """ + return req.headers.get('x-swift-txn-id', + req.headers.get('x-cf-trans-id', default)) + def fallocate(fd, size): """ diff --git a/swift/container/server.py b/swift/container/server.py index 58b09671ed..010e9e0af6 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -31,7 +31,7 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \ from swift.common.db import ContainerBroker from swift.common.utils import get_logger, get_param, hash_path, \ - normalize_timestamp, storage_directory, split_path + normalize_timestamp, storage_directory, split_path, get_txn_id from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.bufferedhttp import http_connect @@ -95,7 +95,7 @@ class ContainerController(object): 'x-delete-timestamp': info['delete_timestamp'], 'x-object-count': info['object_count'], 'x-bytes-used': info['bytes_used'], - 'x-swift-txn-id': req.headers.get('x-swift-txn-id', '-')} + 'x-swift-txn-id': get_txn_id(req, '-')} if req.headers.get('x-account-override-deleted', 'no').lower() == \ 'yes': account_headers['x-account-override-deleted'] = 'yes' @@ -384,7 +384,7 @@ class ContainerController(object): def __call__(self, env, start_response): start_time = time.time() req = Request(env) - self.logger.txn_id = req.headers.get('x-swift-txn-id', None) + self.logger.txn_id = get_txn_id(req, None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: @@ -404,7 +404,7 @@ class ContainerController(object): time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', - req.headers.get('x-swift-txn-id', '-'), + get_txn_id(req, '-'), req.referer or '-', req.user_agent or '-', trans_time) if req.method.upper() == 'REPLICATE': diff --git a/swift/obj/server.py b/swift/obj/server.py index 280ddbf49a..6af3bdd844 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -37,7 +37,7 @@ from eventlet import sleep, Timeout from swift.common.utils import mkdirs, normalize_timestamp, \ storage_directory, hash_path, renamer, fallocate, \ - split_path, drop_buffer_cache, get_logger, write_pickle + split_path, drop_buffer_cache, get_logger, write_pickle, get_txn_id from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, check_mount, \ check_float, check_utf8 @@ -409,7 +409,7 @@ class ObjectController(object): 'x-content-type': file.metadata['Content-Type'], 'x-timestamp': file.metadata['X-Timestamp'], 'x-etag': file.metadata['ETag'], - 'x-swift-txn-id': request.headers.get('x-swift-txn-id', '-')}, + 'x-swift-txn-id': get_txn_id(request, '-')}, device) resp = HTTPCreated(request=request, etag=etag) return resp @@ -531,7 +531,7 @@ class ObjectController(object): file.unlinkold(metadata['X-Timestamp']) self.container_update('DELETE', account, container, obj, request.headers, {'x-timestamp': metadata['X-Timestamp'], - 'x-swift-txn-id': request.headers.get('x-swift-txn-id', '-')}, + 'x-swift-txn-id': get_txn_id(request, '-')}, device) resp = response_class(request=request) return resp @@ -562,7 +562,7 @@ class ObjectController(object): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) - self.logger.txn_id = req.headers.get('x-swift-txn-id', None) + self.logger.txn_id = get_txn_id(req, None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: @@ -583,7 +583,7 @@ class ObjectController(object): time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.referer or '-', - req.headers.get('x-swift-txn-id', '-'), + get_txn_id(req, '-'), req.user_agent or '-', trans_time) if req.method == 'REPLICATE': diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 04187cc82b..d26e626cd9 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -41,7 +41,7 @@ from webob import Request, Response from swift.common.ring import Ring from swift.common.utils import get_logger, normalize_timestamp, split_path, \ - cache_from_env + cache_from_env, get_txn_id from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ check_utf8, CONTAINER_LISTING_LIMIT, MAX_ACCOUNT_NAME_LENGTH, \ @@ -1362,7 +1362,7 @@ class ContainerController(Controller): container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-swift-txn-id': self.trans_id} + 'x-swift-txn-id': self.trans_id} statuses = [] reasons = [] bodies = [] @@ -1707,8 +1707,9 @@ class BaseApplication(object): return HTTPPreconditionFailed(request=req, body='Bad URL') controller = controller(self, **path_parts) - controller.trans_id = req.headers.get('x-swift-txn-id', '-') - self.logger.txn_id = req.headers.get('x-swift-txn-id', None) + txn_id = get_txn_id(req, None) + controller.trans_id = txn_id or '-' + self.logger.txn_id = txn_id try: handler = getattr(controller, req.method) if not getattr(handler, 'publicly_accessible'): @@ -1786,7 +1787,7 @@ class Application(BaseApplication): getattr(req, 'bytes_transferred', 0) or '-', getattr(response, 'bytes_transferred', 0) or '-', req.headers.get('etag', '-'), - req.headers.get('x-swift-txn-id', '-'), + get_txn_id(req, '-'), logged_headers or '-', trans_time, ))) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index b888686660..f8c44440c0 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -28,6 +28,7 @@ from StringIO import StringIO from functools import partial from tempfile import NamedTemporaryFile +from webob import Request from eventlet import sleep from swift.common import utils @@ -80,6 +81,19 @@ class TestUtils(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' + def test_get_txn_id(self): + req = Request.blank('') + req.headers['X-Swift-Txn-Id'] = 'tx12345' + self.assertEquals(utils.get_txn_id(req), 'tx12345') + environ = {'HTTP_X_CF_TRANS_ID': 'tx67890'} + req = Request.blank('', environ=environ) + self.assertEquals(utils.get_txn_id(req), 'tx67890') + req = Request.blank('') + self.assertEquals(utils.get_txn_id(req), None) + self.assertEquals(utils.get_txn_id(req, '-'), '-') + req.headers['X-Cf-Trans-Id'] = 'tx13579' + self.assertEquals(utils.get_txn_id(req, default='test'), 'tx13579') + def test_normalize_timestamp(self): """ Test swift.common.utils.normalize_timestamp """ self.assertEquals(utils.normalize_timestamp('1253327593.48174'), From 9786ab6687e018b3622722e379da68b64cc4af49 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 5 Jan 2011 11:48:58 -0600 Subject: [PATCH 5/6] this is to make gholt happy; well less unhappy --- swift/common/middleware/catch_errors.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/swift/common/middleware/catch_errors.py b/swift/common/middleware/catch_errors.py index f1ab249b25..eee45b8e1f 100644 --- a/swift/common/middleware/catch_errors.py +++ b/swift/common/middleware/catch_errors.py @@ -26,10 +26,10 @@ class CatchErrorMiddleware(object): def __init__(self, app, conf): self.app = app - try: - # if the application already has a logger we should use that one - self.logger = app.logger - except AttributeError: + # if the application already has a logger we should use that one + self.logger = getattr(app, 'logger', None) + if not self.logger: + # and only call get_logger if we have to self.logger = get_logger(conf) def __call__(self, env, start_response): From 30fd2dd0f229d95b53d6efaff4b2792d5f49373b Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 5 Jan 2011 13:52:33 -0600 Subject: [PATCH 6/6] revert x-cf-trans-id rename --- swift/account/server.py | 8 ++++---- swift/common/utils.py | 10 ---------- swift/container/server.py | 8 ++++---- swift/obj/server.py | 10 +++++----- swift/proxy/server.py | 27 +++++++++++++-------------- test/unit/common/test_utils.py | 14 -------------- test/unit/proxy/test_server.py | 5 +---- 7 files changed, 27 insertions(+), 55 deletions(-) diff --git a/swift/account/server.py b/swift/account/server.py index ec9ad343dc..53d604ce93 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -29,7 +29,7 @@ import simplejson from swift.common.db import AccountBroker from swift.common.utils import get_logger, get_param, hash_path, \ - normalize_timestamp, split_path, storage_directory, get_txn_id + normalize_timestamp, split_path, storage_directory from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.db_replicator import ReplicatorRpc @@ -86,7 +86,7 @@ class AccountController(object): return Response(status='507 %s is not mounted' % drive) broker = self._get_account_broker(drive, part, account) if container: # put account container - if get_txn_id(req, None) is None: + if 'x-cf-trans-id' in req.headers: broker.pending_timeout = 3 if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): @@ -296,7 +296,7 @@ class AccountController(object): def __call__(self, env, start_response): start_time = time.time() req = Request(env) - self.logger.txn_id = get_txn_id(req, None) + self.logger.txn_id = req.headers.get('x-cf-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: @@ -319,7 +319,7 @@ class AccountController(object): time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', - get_txn_id(req, '-'), + req.headers.get('x-cf-trans-id', '-'), req.referer or '-', req.user_agent or '-', trans_time, additional_info) diff --git a/swift/common/utils.py b/swift/common/utils.py index d7abac2ec8..c240b242cf 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -107,16 +107,6 @@ def get_param(req, name, default=None): value.decode('utf8') # Ensure UTF8ness return value -def get_txn_id(req, default=None): - """ - Get the transaction id from a request - - :param req: Webob request object - :param default: value to return if no transaction id is found - """ - return req.headers.get('x-swift-txn-id', - req.headers.get('x-cf-trans-id', default)) - def fallocate(fd, size): """ diff --git a/swift/container/server.py b/swift/container/server.py index 010e9e0af6..fc06194de6 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -31,7 +31,7 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \ from swift.common.db import ContainerBroker from swift.common.utils import get_logger, get_param, hash_path, \ - normalize_timestamp, storage_directory, split_path, get_txn_id + normalize_timestamp, storage_directory, split_path from swift.common.constraints import CONTAINER_LISTING_LIMIT, \ check_mount, check_float, check_utf8 from swift.common.bufferedhttp import http_connect @@ -95,7 +95,7 @@ class ContainerController(object): 'x-delete-timestamp': info['delete_timestamp'], 'x-object-count': info['object_count'], 'x-bytes-used': info['bytes_used'], - 'x-swift-txn-id': get_txn_id(req, '-')} + 'x-cf-trans-id': req.headers.get('X-Cf-Trans-Id', '-')} if req.headers.get('x-account-override-deleted', 'no').lower() == \ 'yes': account_headers['x-account-override-deleted'] = 'yes' @@ -384,7 +384,7 @@ class ContainerController(object): def __call__(self, env, start_response): start_time = time.time() req = Request(env) - self.logger.txn_id = get_txn_id(req, None) + self.logger.txn_id = req.headers.get('x-cf-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: @@ -404,7 +404,7 @@ class ContainerController(object): time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', - get_txn_id(req, '-'), + req.headers.get('x-cf-trans-id', '-'), req.referer or '-', req.user_agent or '-', trans_time) if req.method.upper() == 'REPLICATE': diff --git a/swift/obj/server.py b/swift/obj/server.py index 6af3bdd844..7c139d7775 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -37,7 +37,7 @@ from eventlet import sleep, Timeout from swift.common.utils import mkdirs, normalize_timestamp, \ storage_directory, hash_path, renamer, fallocate, \ - split_path, drop_buffer_cache, get_logger, write_pickle, get_txn_id + split_path, drop_buffer_cache, get_logger, write_pickle from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, check_mount, \ check_float, check_utf8 @@ -409,7 +409,7 @@ class ObjectController(object): 'x-content-type': file.metadata['Content-Type'], 'x-timestamp': file.metadata['X-Timestamp'], 'x-etag': file.metadata['ETag'], - 'x-swift-txn-id': get_txn_id(request, '-')}, + 'x-cf-trans-id': request.headers.get('x-cf-trans-id', '-')}, device) resp = HTTPCreated(request=request, etag=etag) return resp @@ -531,7 +531,7 @@ class ObjectController(object): file.unlinkold(metadata['X-Timestamp']) self.container_update('DELETE', account, container, obj, request.headers, {'x-timestamp': metadata['X-Timestamp'], - 'x-swift-txn-id': get_txn_id(request, '-')}, + 'x-cf-trans-id': request.headers.get('x-cf-trans-id', '-')}, device) resp = response_class(request=request) return resp @@ -562,7 +562,7 @@ class ObjectController(object): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) - self.logger.txn_id = get_txn_id(req, None) + self.logger.txn_id = req.headers.get('x-cf-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: @@ -583,7 +583,7 @@ class ObjectController(object): time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.referer or '-', - get_txn_id(req, '-'), + req.headers.get('x-cf-trans-id', '-'), req.user_agent or '-', trans_time) if req.method == 'REPLICATE': diff --git a/swift/proxy/server.py b/swift/proxy/server.py index d26e626cd9..1cde24cfe6 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -41,7 +41,7 @@ from webob import Request, Response from swift.common.ring import Ring from swift.common.utils import get_logger, normalize_timestamp, split_path, \ - cache_from_env, get_txn_id + cache_from_env from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ check_utf8, CONTAINER_LISTING_LIMIT, MAX_ACCOUNT_NAME_LENGTH, \ @@ -356,7 +356,7 @@ class Controller(object): result_code = 0 attempts_left = self.app.account_ring.replica_count path = '/%s' % account - headers = {'x-swift-txn-id': self.trans_id} + headers = {'x-cf-trans-id': self.trans_id} for node in self.iter_nodes(partition, nodes, self.app.account_ring): if self.error_limited(node): continue @@ -430,7 +430,7 @@ class Controller(object): write_acl = None container_size = None attempts_left = self.app.container_ring.replica_count - headers = {'x-swift-txn-id': self.trans_id} + headers = {'x-cf-trans-id': self.trans_id} for node in self.iter_nodes(partition, nodes, self.app.container_ring): if self.error_limited(node): continue @@ -1247,7 +1247,7 @@ class ContainerController(Controller): container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-swift-txn-id': self.trans_id} + 'x-cf-trans-id': self.trans_id} headers.update(value for value in req.headers.iteritems() if value[0].lower() in self.pass_through_headers or value[0].lower().startswith('x-container-meta-')) @@ -1309,7 +1309,7 @@ class ContainerController(Controller): container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-swift-txn-id': self.trans_id} + 'x-cf-trans-id': self.trans_id} headers.update(value for value in req.headers.iteritems() if value[0].lower() in self.pass_through_headers or value[0].lower().startswith('x-container-meta-')) @@ -1362,7 +1362,7 @@ class ContainerController(Controller): container_partition, containers = self.app.container_ring.get_nodes( self.account_name, self.container_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-swift-txn-id': self.trans_id} + 'x-cf-trans-id': self.trans_id} statuses = [] reasons = [] bodies = [] @@ -1450,7 +1450,7 @@ class AccountController(Controller): account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-swift-txn-id': self.trans_id} + 'x-cf-trans-id': self.trans_id} headers.update(value for value in req.headers.iteritems() if value[0].lower().startswith('x-account-meta-')) statuses = [] @@ -1499,7 +1499,7 @@ class AccountController(Controller): account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-swift-txn-id': self.trans_id} + 'X-CF-Trans-Id': self.trans_id} headers.update(value for value in req.headers.iteritems() if value[0].lower().startswith('x-account-meta-')) statuses = [] @@ -1546,7 +1546,7 @@ class AccountController(Controller): account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-swift-txn-id': self.trans_id} + 'X-CF-Trans-Id': self.trans_id} statuses = [] reasons = [] bodies = [] @@ -1683,7 +1683,7 @@ class BaseApplication(object): def update_request(self, req): req.bytes_transferred = '-' req.client_disconnect = False - req.headers['x-swift-txn-id'] = 'tx' + str(uuid.uuid4()) + req.headers['x-cf-trans-id'] = 'tx' + str(uuid.uuid4()) if 'x-storage-token' in req.headers and \ 'x-auth-token' not in req.headers: req.headers['x-auth-token'] = req.headers['x-storage-token'] @@ -1707,9 +1707,8 @@ class BaseApplication(object): return HTTPPreconditionFailed(request=req, body='Bad URL') controller = controller(self, **path_parts) - txn_id = get_txn_id(req, None) - controller.trans_id = txn_id or '-' - self.logger.txn_id = txn_id + controller.trans_id = req.headers.get('x-cf-trans-id', '-') + self.logger.txn_id = req.headers.get('x-cf-trans-id', None) try: handler = getattr(controller, req.method) if not getattr(handler, 'publicly_accessible'): @@ -1787,7 +1786,7 @@ class Application(BaseApplication): getattr(req, 'bytes_transferred', 0) or '-', getattr(response, 'bytes_transferred', 0) or '-', req.headers.get('etag', '-'), - get_txn_id(req, '-'), + req.headers.get('x-cf-trans-id', '-'), logged_headers or '-', trans_time, ))) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index f8c44440c0..b888686660 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -28,7 +28,6 @@ from StringIO import StringIO from functools import partial from tempfile import NamedTemporaryFile -from webob import Request from eventlet import sleep from swift.common import utils @@ -81,19 +80,6 @@ class TestUtils(unittest.TestCase): def setUp(self): utils.HASH_PATH_SUFFIX = 'endcap' - def test_get_txn_id(self): - req = Request.blank('') - req.headers['X-Swift-Txn-Id'] = 'tx12345' - self.assertEquals(utils.get_txn_id(req), 'tx12345') - environ = {'HTTP_X_CF_TRANS_ID': 'tx67890'} - req = Request.blank('', environ=environ) - self.assertEquals(utils.get_txn_id(req), 'tx67890') - req = Request.blank('') - self.assertEquals(utils.get_txn_id(req), None) - self.assertEquals(utils.get_txn_id(req, '-'), '-') - req.headers['X-Cf-Trans-Id'] = 'tx13579' - self.assertEquals(utils.get_txn_id(req, default='test'), 'tx13579') - def test_normalize_timestamp(self): """ Test swift.common.utils.normalize_timestamp """ self.assertEquals(utils.normalize_timestamp('1253327593.48174'), diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index f7e2479a16..4577cd4dac 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -218,7 +218,6 @@ def save_globals(): # tests - class TestController(unittest.TestCase): def setUp(self): @@ -373,7 +372,6 @@ class TestController(unittest.TestCase): test(404, 507, 503) test(503, 503, 503) - class TestProxyServer(unittest.TestCase): def test_unhandled_exception(self): @@ -470,7 +468,6 @@ class TestObjectController(unittest.TestCase): 'text/html', 'text/html'])) test_content_type('test.css', iter(['', '', '', 'text/css', 'text/css', 'text/css'])) - def test_custom_mime_types_files(self): swift_dir = mkdtemp() try: @@ -1718,7 +1715,7 @@ class TestObjectController(unittest.TestCase): for node in nodes: conn = proxy_server.http_connect(node['ip'], node['port'], node['device'], partition, 'PUT', '/a', - {'X-Timestamp': ts, 'x-swift-txn-id': 'test'}) + {'X-Timestamp': ts, 'X-CF-Trans-Id': 'test'}) resp = conn.getresponse() self.assertEquals(resp.status, 201) # Head account, just a double check and really is here to test