x/slogging
Code Issues Proposed changes

Merge pull request #8 from notmyname/robust_stats_log

Browse Source 
made account stats log processing more robust and added tests
This commit is contained in:
David 2011-08-08 09:18:02 -07:00
commit 58f9dccc9c
2 changed files with 31 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
slogging/stats_processor.py
View File

@ -34,15 +34,15 @@ class StatsLogProcessor(object):
(account, (account,
container_count, container_count,
object_count, object_count,
bytes_used) = line.split(',') bytes_used) = line.split(',')[:4]
account = account.strip('"')
container_count = int(container_count.strip('"'))
object_count = int(object_count.strip('"'))
bytes_used = int(bytes_used.strip('"'))
except (IndexError, ValueError): except (IndexError, ValueError):
# bad line data # bad line data
self.logger.debug(_('Bad line data: %s') % repr(line)) self.logger.debug(_('Bad line data: %s') % repr(line))
continue continue
account = account.strip('"')
container_count = int(container_count.strip('"'))
object_count = int(object_count.strip('"'))
bytes_used = int(bytes_used.strip('"'))
aggr_key = (account, year, month, day, hour) aggr_key = (account, year, month, day, hour)
d = account_totals.get(aggr_key, {}) d = account_totals.get(aggr_key, {})
d['replica_count'] = d.setdefault('replica_count', 0) + 1 d['replica_count'] = d.setdefault('replica_count', 0) + 1

30
test_slogging/unit/test_stats_processor.py
View File

@ -13,16 +13,38 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# TODO: Tests
import unittest import unittest
from slogging import stats_processor from slogging import stats_processor
class TestStatsProcessor(unittest.TestCase): class TestStatsProcessor(unittest.TestCase):
def test_placeholder(self): def test_process(self):
pass p = stats_processor.StatsLogProcessor({})
test_obj_stream = ['a, 1, 1, 1', 'a, 1, 1, 1']
res = p.process(test_obj_stream, 'foo', 'bar', '2011/03/14/12/baz')
expected = {('a', '2011', '03', '14', '12'):
{'object_count': 2, 'container_count': 2,
'replica_count': 2, 'bytes_used': 2}}
self.assertEquals(res, expected)
def test_process_extra_columns(self):
p = stats_processor.StatsLogProcessor({})
test_obj_stream = ['a, 1, 1, 1, extra']
res = p.process(test_obj_stream, 'foo', 'bar', '2011/03/14/12/baz')
expected = {('a', '2011', '03', '14', '12'):
{'object_count': 1, 'container_count': 1,
'replica_count': 1, 'bytes_used': 1}}
self.assertEquals(res, expected)
def test_process_bad_line(self):
p = stats_processor.StatsLogProcessor({})
test_obj_stream = ['a, 1, 1, 1, extra', '', 'some bad line']
res = p.process(test_obj_stream, 'foo', 'bar', '2011/03/14/12/baz')
expected = {('a', '2011', '03', '14', '12'):
{'object_count': 1, 'container_count': 1,
'replica_count': 1, 'bytes_used': 1}}
self.assertEquals(res, expected)
if __name__ == '__main__': if __name__ == '__main__':