Merge pull request #36 from dpgoetz/master

adding in changes for swift.source

etc/log-processor.conf-sample

@@ -31,6 +31,9 @@ class_path = slogging.access_processor.AccessLogProcessor
 # server_name = proxy-server
 # user = swift
 # warn_percent = 0.8
+# list of swift.sources (see swift/proxy/server.py posthooklogger)
+# that count as service traffic
+# service_log_sources =
 
 [log-processor-stats]
 # log_dir = /var/log/swift/

slogging/access_processor.py

@@ -29,12 +29,11 @@ class AccessLogProcessor(object):
 
     def __init__(self, conf):
         self.server_name = conf.get('server_name', 'proxy-server')
-        self.lb_private_ips = [x.strip() for x in \
+        for conf_tag in ['lb_private_ips', 'service_ips',
-                               conf.get('lb_private_ips', '').split(',')\
+                         'service_log_sources']:
-                               if x.strip()]
+            setattr(self, conf_tag,
-        self.service_ips = [x.strip() for x in \
+                [x.strip() for x in conf.get(conf_tag, '').split(',') \
-                            conf.get('service_ips', '').split(',')\
+                 if x.strip()])
-                            if x.strip()]
         self.warn_percent = float(conf.get('warn_percent', '0.8'))
         self.logger = get_logger(conf, log_route='access-processor')
 
@@ -42,6 +41,8 @@ class AccessLogProcessor(object):
         '''given a raw access log line, return a dict of the good parts'''
         d = {}
         try:
+            log_source = None
+            split_log = raw_log[16:].split(' ')
             (unused,
             server,
             client_ip,
@@ -59,8 +60,9 @@ class AccessLogProcessor(object):
             etag,
             trans_id,
             headers,
-            processing_time) = (unquote(x) for x in
+            processing_time) = (unquote(x) for x in split_log[:18])
-                                raw_log[16:].split(' ')[:18])
+            if len(split_log) > 18:
+                log_source = split_log[18]
         except ValueError:
             self.logger.debug(_('Bad line data: %s') % repr(raw_log))
             return {}
@@ -130,6 +132,7 @@ class AccessLogProcessor(object):
         d['bytes_out'] = int(d['bytes_out'].replace('-', '0'))
         d['bytes_in'] = int(d['bytes_in'].replace('-', '0'))
         d['code'] = int(d['code'])
+        d['log_source'] = log_source
         return d
 
     def process(self, obj_stream, data_object_account, data_object_container,
@@ -167,14 +170,13 @@ class AccessLogProcessor(object):
 
             aggr_key = (account, year, month, day, hour)
             d = hourly_aggr_info.get(aggr_key, {})
-            if line_data['lb_ip'] in self.lb_private_ips:
+            if line_data['lb_ip'] in self.lb_private_ips or \
+                    line_data['client_ip'] in self.service_ips or \
+                    line_data['log_source'] in self.service_log_sources:
                 source = 'service'
             else:
                 source = 'public'
 
-            if line_data['client_ip'] in self.service_ips:
-                source = 'service'
-
             d[(source, 'bytes_out')] = d.setdefault((
                 source, 'bytes_out'), 0) + bytes_out
             d[(source, 'bytes_in')] = d.setdefault((source, 'bytes_in'), 0) + \

test_slogging/unit/test_access_processor.py

@@ -39,7 +39,8 @@ class TestAccessProcessor(unittest.TestCase):
                     'bytes_out': 13, 'container_name': 'c', 'day': '1',
                     'minute': '5', 'account': 'a', 'hour': '4',
                     'referrer': '9', 'request': '/v1/a/c/o',
-                    'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
+                    'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3',
+                    'log_source': None}
         for param in access_processor.LISTING_PARAMS:
             expected[param] = 1
         expected['query'] = query
@@ -70,7 +71,8 @@ class TestAccessProcessor(unittest.TestCase):
                     'bytes_out': 13, 'container_name': 'c', 'day': '1',
                     'minute': '5', 'account': 'a', 'hour': '4',
                     'referrer': '9', 'request': '/v1/a/c/o',
-                    'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
+                    'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3',
+                    'log_source': None,}
         self.assertEquals(res, expected)
         # too many fields
         log_line = [str(x) for x in range(19)]
@@ -86,7 +88,8 @@ class TestAccessProcessor(unittest.TestCase):
                     'bytes_out': 13, 'container_name': 'c', 'day': '1',
                     'minute': '5', 'account': 'a', 'hour': '4',
                     'referrer': '9', 'request': '/v1/a/c/o',
-                    'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
+                    'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3',
+                    'log_source': '18',}
         self.assertEquals(res, expected)
 
 

test_slogging/unit/test_log_processor.py

@@ -164,7 +164,8 @@ use = egg:swift#proxy
            'request': '/v1/acct/foo/bar',
            'user_agent': 'curl',
            'bytes_in': 6,
-           'lb_ip': '4.5.6.7'})
+           'lb_ip': '4.5.6.7',
+            'log_source': None})
 
     def test_process_one_access_file(self):
         access_proxy_config = self.proxy_config.copy()