From 4cb4f57f7cf3a4c81f255367cfb48361cf44c04d Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Thu, 4 Aug 2011 14:19:24 -0500 Subject: [PATCH] made account stats log processing more robust and added tests --- slogging/stats_processor.py | 10 ++++---- test_slogging/unit/test_stats_processor.py | 30 +++++++++++++++++++--- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/slogging/stats_processor.py b/slogging/stats_processor.py index f9496c1..4dc87c5 100644 --- a/slogging/stats_processor.py +++ b/slogging/stats_processor.py @@ -34,15 +34,15 @@ class StatsLogProcessor(object): (account, container_count, object_count, - bytes_used) = line.split(',') + bytes_used) = line.split(',')[:4] + account = account.strip('"') + container_count = int(container_count.strip('"')) + object_count = int(object_count.strip('"')) + bytes_used = int(bytes_used.strip('"')) except (IndexError, ValueError): # bad line data self.logger.debug(_('Bad line data: %s') % repr(line)) continue - account = account.strip('"') - container_count = int(container_count.strip('"')) - object_count = int(object_count.strip('"')) - bytes_used = int(bytes_used.strip('"')) aggr_key = (account, year, month, day, hour) d = account_totals.get(aggr_key, {}) d['replica_count'] = d.setdefault('replica_count', 0) + 1 diff --git a/test_slogging/unit/test_stats_processor.py b/test_slogging/unit/test_stats_processor.py index 10ec6b9..f199754 100644 --- a/test_slogging/unit/test_stats_processor.py +++ b/test_slogging/unit/test_stats_processor.py @@ -13,16 +13,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: Tests - import unittest from slogging import stats_processor class TestStatsProcessor(unittest.TestCase): - def test_placeholder(self): - pass + def test_process(self): + p = stats_processor.StatsLogProcessor({}) + test_obj_stream = ['a, 1, 1, 1', 'a, 1, 1, 1'] + res = p.process(test_obj_stream, 'foo', 'bar', '2011/03/14/12/baz') + expected = {('a', '2011', '03', '14', '12'): + {'object_count': 2, 'container_count': 2, + 'replica_count': 2, 'bytes_used': 2}} + self.assertEquals(res, expected) + + def test_process_extra_columns(self): + p = stats_processor.StatsLogProcessor({}) + test_obj_stream = ['a, 1, 1, 1, extra'] + res = p.process(test_obj_stream, 'foo', 'bar', '2011/03/14/12/baz') + expected = {('a', '2011', '03', '14', '12'): + {'object_count': 1, 'container_count': 1, + 'replica_count': 1, 'bytes_used': 1}} + self.assertEquals(res, expected) + + def test_process_bad_line(self): + p = stats_processor.StatsLogProcessor({}) + test_obj_stream = ['a, 1, 1, 1, extra', '', 'some bad line'] + res = p.process(test_obj_stream, 'foo', 'bar', '2011/03/14/12/baz') + expected = {('a', '2011', '03', '14', '12'): + {'object_count': 1, 'container_count': 1, + 'replica_count': 1, 'bytes_used': 1}} + self.assertEquals(res, expected) if __name__ == '__main__':