x/slogging
Code Issues Proposed changes

Merge pull request #36 from dpgoetz/master

Browse Source 
adding in changes for swift.source
This commit is contained in:
John Dickinson 2012-03-15 13:52:52 -07:00
commit 856ecacc7c
4 changed files with 25 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
etc/log-processor.conf-sample
View File

@ -31,6 +31,9 @@ class_path = slogging.access_processor.AccessLogProcessor
# server_name = proxy-server
# user = swift
# warn_percent = 0.8
# list of swift.sources (see swift/proxy/server.py posthooklogger)
# that count as service traffic
# service_log_sources =
[log-processor-stats]
# log_dir = /var/log/swift/

26
slogging/access_processor.py
View File

@ -29,12 +29,11 @@ class AccessLogProcessor(object):
def __init__(self, conf):
self.server_name = conf.get('server_name', 'proxy-server')
self.lb_private_ips = [x.strip() for x in \
conf.get('lb_private_ips', '').split(',')\
if x.strip()]
self.service_ips = [x.strip() for x in \
conf.get('service_ips', '').split(',')\
if x.strip()]
for conf_tag in ['lb_private_ips', 'service_ips',
'service_log_sources']:
setattr(self, conf_tag,
[x.strip() for x in conf.get(conf_tag, '').split(',') \
if x.strip()])
self.warn_percent = float(conf.get('warn_percent', '0.8'))
self.logger = get_logger(conf, log_route='access-processor')
@ -42,6 +41,8 @@ class AccessLogProcessor(object):
'''given a raw access log line, return a dict of the good parts'''
d = {}
try:
log_source = None
split_log = raw_log[16:].split(' ')
(unused,
server,
client_ip,
@ -59,8 +60,9 @@ class AccessLogProcessor(object):
etag,
trans_id,
headers,
processing_time) = (unquote(x) for x in
raw_log[16:].split(' ')[:18])
processing_time) = (unquote(x) for x in split_log[:18])
if len(split_log) > 18:
log_source = split_log[18]
except ValueError:
self.logger.debug(_('Bad line data: %s') % repr(raw_log))
return {}
@ -130,6 +132,7 @@ class AccessLogProcessor(object):
d['bytes_out'] = int(d['bytes_out'].replace('-', '0'))
d['bytes_in'] = int(d['bytes_in'].replace('-', '0'))
d['code'] = int(d['code'])
d['log_source'] = log_source
return d
def process(self, obj_stream, data_object_account, data_object_container,
@ -167,14 +170,13 @@ class AccessLogProcessor(object):
aggr_key = (account, year, month, day, hour)
d = hourly_aggr_info.get(aggr_key, {})
if line_data['lb_ip'] in self.lb_private_ips:
if line_data['lb_ip'] in self.lb_private_ips or \
line_data['client_ip'] in self.service_ips or \
line_data['log_source'] in self.service_log_sources:
source = 'service'
else:
source = 'public'
if line_data['client_ip'] in self.service_ips:
source = 'service'
d[(source, 'bytes_out')] = d.setdefault((
source, 'bytes_out'), 0) + bytes_out
d[(source, 'bytes_in')] = d.setdefault((source, 'bytes_in'), 0) + \

9
test_slogging/unit/test_access_processor.py
View File

@ -39,7 +39,8 @@ class TestAccessProcessor(unittest.TestCase):
'bytes_out': 13, 'container_name': 'c', 'day': '1',
'minute': '5', 'account': 'a', 'hour': '4',
'referrer': '9', 'request': '/v1/a/c/o',
'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3',
'log_source': None}
for param in access_processor.LISTING_PARAMS:
expected[param] = 1
expected['query'] = query
@ -70,7 +71,8 @@ class TestAccessProcessor(unittest.TestCase):
'bytes_out': 13, 'container_name': 'c', 'day': '1',
'minute': '5', 'account': 'a', 'hour': '4',
'referrer': '9', 'request': '/v1/a/c/o',
'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3',
'log_source': None,}
self.assertEquals(res, expected)
# too many fields
log_line = [str(x) for x in range(19)]
@ -86,7 +88,8 @@ class TestAccessProcessor(unittest.TestCase):
'bytes_out': 13, 'container_name': 'c', 'day': '1',
'minute': '5', 'account': 'a', 'hour': '4',
'referrer': '9', 'request': '/v1/a/c/o',
'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3',
'log_source': '18',}
self.assertEquals(res, expected)

3
test_slogging/unit/test_log_processor.py
View File

@ -164,7 +164,8 @@ use = egg:swift#proxy
'request': '/v1/acct/foo/bar',
'user_agent': 'curl',
'bytes_in': 6,
'lb_ip': '4.5.6.7'})
'lb_ip': '4.5.6.7',
'log_source': None})
def test_process_one_access_file(self):
access_proxy_config = self.proxy_config.copy()