openstack/swift
Code Issues Proposed changes

improved the container listings for the stats processor

Browse Source
This commit is contained in:
John Dickinson 2010-11-24 18:02:31 +00:00 committed by Tarmac
commit f3f62520a0
2 changed files with 27 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
swift/stats/log_processor.py
View File

@ -113,48 +113,35 @@ class LogProcessor(object):
listing_filter=None):
'''
Get a container listing, filtered by start_date, end_date, and
listing_filter. Dates, if given, should be in YYYYMMDDHH format
listing_filter. Dates, if given, must be in YYYYMMDDHH format
'''
search_key = None
if start_date is not None:
date_parts = []
try:
year, start_date = start_date[:4], start_date[4:]
if year:
date_parts.append(year)
month, start_date = start_date[:2], start_date[2:]
if month:
date_parts.append(month)
day, start_date = start_date[:2], start_date[2:]
if day:
date_parts.append(day)
hour, start_date = start_date[:2], start_date[2:]
if hour:
date_parts.append(hour)
except IndexError:
parsed_date = time.strptime(start_date, '%Y%m%d%H')
except ValueError:
pass
else:
search_key = '/'.join(date_parts)
year = '%04d' % parsed_date.tm_year
month = '%02d' % parsed_date.tm_mon
day = '%02d' % parsed_date.tm_mday
hour = '%02d' % parsed_date.tm_hour
search_key = '/'.join([year, month, day, hour])
end_key = None
if end_date is not None:
date_parts = []
try:
year, end_date = end_date[:4], end_date[4:]
if year:
date_parts.append(year)
month, end_date = end_date[:2], end_date[2:]
if month:
date_parts.append(month)
day, end_date = end_date[:2], end_date[2:]
if day:
date_parts.append(day)
hour, end_date = end_date[:2], end_date[2:]
if hour:
date_parts.append(hour)
except IndexError:
parsed_date = time.strptime(end_date, '%Y%m%d%H')
except ValueError:
pass
else:
end_key = '/'.join(date_parts)
year = '%04d' % parsed_date.tm_year
month = '%02d' % parsed_date.tm_mon
day = '%02d' % parsed_date.tm_mday
hour = '%02d' % parsed_date.tm_hour
# Since the end_marker filters by <=, we need to add something
# to then end_key to make sure we get all the data under the
# last hour. Adding '/\x7f' should be all inclusive.
end_key = '/'.join([year, month, day, hour]) + '/\x7f'
container_listing = self.internal_proxy.get_container_list(
swift_account,
container_name,

10
test/unit/stats/test_log_processor.py
View File

@ -37,9 +37,10 @@ class DumbInternalProxy(object):
end_marker=None):
n = '2010/03/14/13/obj1'
if marker is None or n > marker:
if end_marker and n <= end_marker:
if end_marker:
if n <= end_marker:
return [{'name': n}]
elif end_marker:
else:
return []
return [{'name': n}]
return []
@ -178,6 +179,10 @@ use = egg:swift#proxy
end_date='2010031412')
expected = []
self.assertEquals(result, expected)
result = p.get_container_listing('a', 'foo', start_date='2010031412',
end_date='2010031413')
expected = ['2010/03/14/13/obj1']
self.assertEquals(result, expected)
def test_get_object_data(self):
p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
@ -212,7 +217,6 @@ use = egg:swift#proxy
p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
result = p.generate_keylist_mapping()
expected = {}
print p.plugins
self.assertEquals(result, expected)
def test_generate_keylist_mapping_with_dummy_plugins(self):