Merge remote-tracking branch 'rackspace/master'

etc/log-processor.conf-sample

@@ -33,7 +33,6 @@ class_path = slogging.access_processor.AccessLogProcessor
 # warn_percent = 0.8
 # list of swift.sources (see swift/proxy/server.py posthooklogger)
 # that count as service traffic
-# service_log_sources =
 # content_type =
 
 [log-processor-stats]

slogging/access_log_delivery.py

@@ -144,6 +144,8 @@ class AccessLogDelivery(LogProcessorCommon):
 
     def convert_log_line(self, raw_log):
         parts = self.log_line_parser(raw_log)
+        if parts == {}:
+            return None, None, None
         return (make_clf_from_parts(parts),
                 parts.get('account'),
                 parts.get('container_name'))
@@ -152,6 +154,12 @@ class AccessLogDelivery(LogProcessorCommon):
         '''given a raw access log line, return a dict of the good parts'''
         d = {}
         try:
+            log_arr = raw_log[16:].split(' ')
+            if len(log_arr) > 18:
+                log_source = log_arr[18]
+                if log_source != '-':
+                    # internal proxy log
+                    return {}
             (unused,
             server,
             client_ip,
@@ -169,8 +177,7 @@ class AccessLogDelivery(LogProcessorCommon):
             etag,
             trans_id,
             headers,
-            processing_time) = (unquote(x) for x in
+            processing_time) = (unquote(x) for x in log_arr[:18])
-                                raw_log[16:].split(' ')[:18])
         except ValueError:
             self.logger.debug(_('Bad line data: %s') % repr(raw_log))
             return {}
@@ -307,7 +314,10 @@ class AccessLogDeliveryDaemon(Daemon):
     def run_forever(self, *a, **kw):
         while True:
             start_time = time.time()
-            self.run_once()
+            try:
+                self.run_once()
+            except Exception:
+                self.logger.exception('Run once failed')
             end_time = time.time()
             # don't run more than once every self.frequency seconds
             sleep_time = self.frequency - (end_time - start_time)

slogging/access_processor.py

@@ -50,8 +50,7 @@ class AccessLogProcessor(object):
 
     def __init__(self, conf):
         self.server_name = conf.get('server_name', 'proxy-server')
-        for conf_tag in ['lb_private_ips', 'service_ips',
+        for conf_tag in ['lb_private_ips', 'service_ips']:
-                         'service_log_sources']:
             setattr(self, conf_tag, return_ips(conf, conf_tag))
         self.warn_percent = float(conf.get('warn_percent', '0.8'))
         self.logger = get_logger(conf, log_route='access-processor')
@@ -193,7 +192,7 @@ class AccessLogProcessor(object):
                 sanitize_ips(line_data)
             if line_data['lb_ip'] in self.lb_private_ips or \
                     line_data['client_ip'] in self.service_ips or \
-                    line_data['log_source'] in self.service_log_sources:
+                    line_data['log_source'] not in ['-', None]:
                 source = 'service'
             else:
                 source = 'public'

test_slogging/unit/test_access_log_delivery.py

@@ -147,15 +147,8 @@ class TestAccessLogDelivery(unittest.TestCase):
         log_line[6] = '/v1/a/c/o'
         log_line = 'x' * 16 + ' '.join(log_line)
         res = p.log_line_parser(log_line)
-        expected = {'code': 8, 'processing_time': '17', 'auth_token': '11',
+        # throws away invalid log lines
-                    'month': '01', 'second': '6', 'year': '3', 'tz': '+0000',
+        self.assertEquals(res, {})
-                    'http_version': '7', 'object_name': 'o', 'etag': '14',
-                    'method': '5', 'trans_id': '15', 'client_ip': '2',
-                    'bytes_out': 13, 'container_name': 'c', 'day': '1',
-                    'minute': '5', 'account': 'a', 'hour': '4',
-                    'referrer': '9', 'request': '/v1/a/c/o',
-                    'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
-        self.assertEquals(res, expected)
 
     def test_make_clf_from_parts(self):
         p = access_log_delivery.AccessLogDelivery(self.conf, DumbLogger())

test_slogging/unit/test_log_processor.py

@@ -167,7 +167,7 @@ use = egg:swift#proxy
            'user_agent': 'curl',
            'bytes_in': 6,
            'lb_ip': '4.5.6.7',
-            'log_source': None})
+           'log_source': None})
 
     def test_process_one_access_file(self):
         access_proxy_config = self.proxy_config.copy()
@@ -736,3 +736,7 @@ class TestLogProcessorDaemon(unittest.TestCase):
                     'Method should not be called'
 
         MockLogProcessorDaemon(self).run_once()
+
+
+if __name__ == '__main__':
+    unittest.main()