x/slogging
Code Issues Proposed changes

Changed the dict to a set, thus removing the need for

Browse Source 
the sentinal value. Also included the speed difference
test, but it's in a seperate folder and not part of
the normal test-run (for expediancy's sake)
This commit is contained in:
Stef T 2012-10-04 19:18:11 +02:00
parent 9ce73e2e1e
commit 6d8ada8447
2 changed files with 62 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
slogging/access_processor.py
View File

@ -24,7 +24,7 @@ try:
CIDR_support = True
def return_ips(conf, conf_tag):
return dict((k, 1) for k in
return set(k for k in
IpRangeList(*[x.strip() for x in
conf.get(conf_tag, '').split(',') if x.strip()]))
def sanitize_ips(line_data):

61
test_slogging/speed/test_cidr_speed.py Executable file
View File

@ -0,0 +1,61 @@
#!/usr/bin/env python
import iptools
import datetime
import unittest
from slogging import access_processor
class TestAccessProcessorSpeed(unittest.TestCase):
def test_CIDR_speed(self):
line = 'Sep 16 20:00:02 srv testsrv 192.%s.119.%s - ' \
'16/Sep/2012/20/00/02 GET /v1/a/c/o HTTP/1.0 ' \
'200 - StaticWeb - - 17005 - txn - 0.0095 -'
ips1 = iptools.IpRangeList(*[x.strip() for x in
'127.0.0.1,192.168/16,10/24'.split(',')
if x.strip()])
ips2 = iptools.IpRangeList(*[x.strip() for x in
'172.168/16,10/30'.split(',')
if x.strip()])
ips3 = iptools.IpRangeList(*[x.strip() for x in
'127.0.0.1,11/24'.split(',')
if x.strip()])
orig_start = datetime.datetime.utcnow()
hit = 0
for n in range(255):
for a in range(255):
stream = line % (n, a)
data = stream.split(" ")
if data[5] in ips1 or data[5] in ips2 or data[5] in ips3:
hit += 1
orig_end = datetime.datetime.utcnow()
orig_secs = float("%d.%d" % ((orig_end - orig_start).seconds,
(orig_end - orig_start).microseconds))
self.assertEqual(hit, 255)
# now, let's check the speed with pure dicts
dict1 = set(k for k in
iptools.IpRangeList(*[x.strip() for x in
'127.0.0.1,192.168/16,10/24'.split(',') if x.strip()]))
dict2 = set(k for k in
iptools.IpRangeList(*[x.strip() for x in
'172.168/16,10/30'.split(',') if x.strip()]))
dict3 = set(k for k in
iptools.IpRangeList(*[x.strip() for x in
'127.0.0.1,11/24'.split(',') if x.strip()]))
new_start = datetime.datetime.utcnow()
hit = 0
for n in range(255):
for a in range(255):
stream = line % (n, a)
data = stream.split(" ")
if data[5] in dict1 or data[5] in dict2 or data[5] in dict3:
hit += 1
new_end = datetime.datetime.utcnow()
new_secs = float("%d.%d" % ((new_end - new_start).seconds,
(new_end - new_start).microseconds))
self.assertEqual(hit, 255)
self.assertTrue(new_secs < orig_secs)