diff --git a/etc/log-processor.conf-sample b/etc/log-processor.conf-sample index 142e96f..a119bcd 100644 --- a/etc/log-processor.conf-sample +++ b/etc/log-processor.conf-sample @@ -33,7 +33,6 @@ class_path = slogging.access_processor.AccessLogProcessor # warn_percent = 0.8 # list of swift.sources (see swift/proxy/server.py posthooklogger) # that count as service traffic -# service_log_sources = # content_type = [log-processor-stats] diff --git a/slogging/access_log_delivery.py b/slogging/access_log_delivery.py index 34034b3..2ae0f01 100644 --- a/slogging/access_log_delivery.py +++ b/slogging/access_log_delivery.py @@ -144,6 +144,8 @@ class AccessLogDelivery(LogProcessorCommon): def convert_log_line(self, raw_log): parts = self.log_line_parser(raw_log) + if parts == {}: + return None, None, None return (make_clf_from_parts(parts), parts.get('account'), parts.get('container_name')) @@ -152,6 +154,12 @@ class AccessLogDelivery(LogProcessorCommon): '''given a raw access log line, return a dict of the good parts''' d = {} try: + log_arr = raw_log[16:].split(' ') + if len(log_arr) > 18: + log_source = log_arr[18] + if log_source != '-': + # internal proxy log + return {} (unused, server, client_ip, @@ -169,8 +177,7 @@ class AccessLogDelivery(LogProcessorCommon): etag, trans_id, headers, - processing_time) = (unquote(x) for x in - raw_log[16:].split(' ')[:18]) + processing_time) = (unquote(x) for x in log_arr[:18]) except ValueError: self.logger.debug(_('Bad line data: %s') % repr(raw_log)) return {} diff --git a/slogging/access_processor.py b/slogging/access_processor.py index 193dddd..efd3d94 100644 --- a/slogging/access_processor.py +++ b/slogging/access_processor.py @@ -50,8 +50,7 @@ class AccessLogProcessor(object): def __init__(self, conf): self.server_name = conf.get('server_name', 'proxy-server') - for conf_tag in ['lb_private_ips', 'service_ips', - 'service_log_sources']: + for conf_tag in ['lb_private_ips', 'service_ips']: setattr(self, conf_tag, return_ips(conf, conf_tag)) self.warn_percent = float(conf.get('warn_percent', '0.8')) self.logger = get_logger(conf, log_route='access-processor') @@ -193,7 +192,7 @@ class AccessLogProcessor(object): sanitize_ips(line_data) if line_data['lb_ip'] in self.lb_private_ips or \ line_data['client_ip'] in self.service_ips or \ - line_data['log_source'] in self.service_log_sources: + line_data['log_source'] != '-': source = 'service' else: source = 'public'