From acc5e179aad6f1b8deed1e4ef6acbe01839e8c7d Mon Sep 17 00:00:00 2001 From: Greg Lange Date: Fri, 14 Dec 2012 19:27:02 +0000 Subject: [PATCH 1/3] added try/accept in access log delivery --- slogging/access_log_delivery.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/slogging/access_log_delivery.py b/slogging/access_log_delivery.py index 7fc35c6..34034b3 100644 --- a/slogging/access_log_delivery.py +++ b/slogging/access_log_delivery.py @@ -307,7 +307,10 @@ class AccessLogDeliveryDaemon(Daemon): def run_forever(self, *a, **kw): while True: start_time = time.time() - self.run_once() + try: + self.run_once() + except Exception: + self.logger.exception('Run once failed') end_time = time.time() # don't run more than once every self.frequency seconds sleep_time = self.frequency - (end_time - start_time) From afc82470050290172e0dbf5ccf5c319f7edd1f65 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Fri, 8 Feb 2013 14:16:16 -0800 Subject: [PATCH 2/3] ignore proxy logs with any swift.source --- etc/log-processor.conf-sample | 1 - slogging/access_log_delivery.py | 11 +++++++++-- slogging/access_processor.py | 5 ++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/etc/log-processor.conf-sample b/etc/log-processor.conf-sample index 142e96f..a119bcd 100644 --- a/etc/log-processor.conf-sample +++ b/etc/log-processor.conf-sample @@ -33,7 +33,6 @@ class_path = slogging.access_processor.AccessLogProcessor # warn_percent = 0.8 # list of swift.sources (see swift/proxy/server.py posthooklogger) # that count as service traffic -# service_log_sources = # content_type = [log-processor-stats] diff --git a/slogging/access_log_delivery.py b/slogging/access_log_delivery.py index 34034b3..2ae0f01 100644 --- a/slogging/access_log_delivery.py +++ b/slogging/access_log_delivery.py @@ -144,6 +144,8 @@ class AccessLogDelivery(LogProcessorCommon): def convert_log_line(self, raw_log): parts = self.log_line_parser(raw_log) + if parts == {}: + return None, None, None return (make_clf_from_parts(parts), parts.get('account'), parts.get('container_name')) @@ -152,6 +154,12 @@ class AccessLogDelivery(LogProcessorCommon): '''given a raw access log line, return a dict of the good parts''' d = {} try: + log_arr = raw_log[16:].split(' ') + if len(log_arr) > 18: + log_source = log_arr[18] + if log_source != '-': + # internal proxy log + return {} (unused, server, client_ip, @@ -169,8 +177,7 @@ class AccessLogDelivery(LogProcessorCommon): etag, trans_id, headers, - processing_time) = (unquote(x) for x in - raw_log[16:].split(' ')[:18]) + processing_time) = (unquote(x) for x in log_arr[:18]) except ValueError: self.logger.debug(_('Bad line data: %s') % repr(raw_log)) return {} diff --git a/slogging/access_processor.py b/slogging/access_processor.py index 193dddd..efd3d94 100644 --- a/slogging/access_processor.py +++ b/slogging/access_processor.py @@ -50,8 +50,7 @@ class AccessLogProcessor(object): def __init__(self, conf): self.server_name = conf.get('server_name', 'proxy-server') - for conf_tag in ['lb_private_ips', 'service_ips', - 'service_log_sources']: + for conf_tag in ['lb_private_ips', 'service_ips']: setattr(self, conf_tag, return_ips(conf, conf_tag)) self.warn_percent = float(conf.get('warn_percent', '0.8')) self.logger = get_logger(conf, log_route='access-processor') @@ -193,7 +192,7 @@ class AccessLogProcessor(object): sanitize_ips(line_data) if line_data['lb_ip'] in self.lb_private_ips or \ line_data['client_ip'] in self.service_ips or \ - line_data['log_source'] in self.service_log_sources: + line_data['log_source'] != '-': source = 'service' else: source = 'public' From 61ed362971d1dd77769b682f9847b2a8abb5459f Mon Sep 17 00:00:00 2001 From: David Goetz Date: Mon, 11 Feb 2013 10:50:04 -0800 Subject: [PATCH 3/3] fix unit tests --- slogging/access_processor.py | 2 +- test_slogging/unit/test_access_log_delivery.py | 11 ++--------- test_slogging/unit/test_log_processor.py | 6 +++++- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/slogging/access_processor.py b/slogging/access_processor.py index efd3d94..2bf4f91 100644 --- a/slogging/access_processor.py +++ b/slogging/access_processor.py @@ -192,7 +192,7 @@ class AccessLogProcessor(object): sanitize_ips(line_data) if line_data['lb_ip'] in self.lb_private_ips or \ line_data['client_ip'] in self.service_ips or \ - line_data['log_source'] != '-': + line_data['log_source'] not in ['-', None]: source = 'service' else: source = 'public' diff --git a/test_slogging/unit/test_access_log_delivery.py b/test_slogging/unit/test_access_log_delivery.py index a5e2be5..2182fea 100644 --- a/test_slogging/unit/test_access_log_delivery.py +++ b/test_slogging/unit/test_access_log_delivery.py @@ -147,15 +147,8 @@ class TestAccessLogDelivery(unittest.TestCase): log_line[6] = '/v1/a/c/o' log_line = 'x' * 16 + ' '.join(log_line) res = p.log_line_parser(log_line) - expected = {'code': 8, 'processing_time': '17', 'auth_token': '11', - 'month': '01', 'second': '6', 'year': '3', 'tz': '+0000', - 'http_version': '7', 'object_name': 'o', 'etag': '14', - 'method': '5', 'trans_id': '15', 'client_ip': '2', - 'bytes_out': 13, 'container_name': 'c', 'day': '1', - 'minute': '5', 'account': 'a', 'hour': '4', - 'referrer': '9', 'request': '/v1/a/c/o', - 'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'} - self.assertEquals(res, expected) + # throws away invalid log lines + self.assertEquals(res, {}) def test_make_clf_from_parts(self): p = access_log_delivery.AccessLogDelivery(self.conf, DumbLogger()) diff --git a/test_slogging/unit/test_log_processor.py b/test_slogging/unit/test_log_processor.py index b1cb6c5..cb9b471 100644 --- a/test_slogging/unit/test_log_processor.py +++ b/test_slogging/unit/test_log_processor.py @@ -167,7 +167,7 @@ use = egg:swift#proxy 'user_agent': 'curl', 'bytes_in': 6, 'lb_ip': '4.5.6.7', - 'log_source': None}) + 'log_source': None}) def test_process_one_access_file(self): access_proxy_config = self.proxy_config.copy() @@ -736,3 +736,7 @@ class TestLogProcessorDaemon(unittest.TestCase): 'Method should not be called' MockLogProcessorDaemon(self).run_once() + + +if __name__ == '__main__': + unittest.main()