update unittest for file_matcher and line_matcher

Change-Id: I5b9fe394b42239c194c2ab083cd487e0a097300a

compass/tests/log_analyzor/data/sample_log

@@ -0,0 +1,3 @@
+Line1
+Line2
+Line3

compass/tests/log_analyzor/test_file_matcher.py

@@ -15,96 +15,21 @@
 """test file matcher module"""
 
 import datetime
-import mock
 import os
 import unittest2
 
 os.environ['COMPASS_IGNORE_SETTING'] = 'true'
 
-
 from compass.utils import setting_wrapper as setting
 reload(setting)
 
-
-from compass.db import database
-from compass.db.model import Adapter
-from compass.db.model import Cluster
-from compass.db.model import ClusterHost
-from compass.db.model import ClusterState
-from compass.db.model import HostState
-from compass.db.model import LogProgressingHistory
-from compass.db.model import Machine
-from compass.db.model import Role
-from compass.db.model import Switch
-
 from compass.log_analyzor import file_matcher
-
-from compass.log_analyzor.line_matcher import IncrementalProgress
 from compass.log_analyzor.line_matcher import LineMatcher
-from compass.log_analyzor.line_matcher import Progress
 
 from compass.utils import flags
 from compass.utils import logsetting
 
 
-def prepare_database(config):
-        with database.session() as session:
-            adapters = {}
-            for adapter_config in config['ADAPTERS']:
-                adapter = Adapter(**adapter_config)
-                session.add(adapter)
-                adapters[adapter_config['name']] = adapter
-
-            roles = {}
-            for role_config in config['ROLES']:
-                role = Role(**role_config)
-                session.add(role)
-                roles[role_config['name']] = role
-
-            switches = {}
-            for switch_config in config['SWITCHES']:
-                switch = Switch(**switch_config)
-                session.add(switch)
-                switches[switch_config['ip']] = switch
-
-            machines = {}
-            for switch_ip, machine_configs in (
-                config['MACHINES_BY_SWITCH'].items()
-            ):
-                for machine_config in machine_configs:
-                    machine = Machine(**machine_config)
-                    machines[machine_config['mac']] = machine
-                    machine.switch = switches[switch_ip]
-                    session.add(machine)
-
-            clusters = {}
-            for cluster_config in config['CLUSTERS']:
-                adapter_name = cluster_config['adapter']
-                del cluster_config['adapter']
-                cluster = Cluster(**cluster_config)
-                clusters[cluster_config['name']] = cluster
-                cluster.adapter = adapters[adapter_name]
-                cluster.state = ClusterState(
-                    state="INSTALLING", progress=0.0, message='')
-                session.add(cluster)
-
-            hosts = {}
-            for cluster_name, host_configs in (
-                config['HOSTS_BY_CLUSTER'].items()
-            ):
-                for host_config in host_configs:
-                    mac = host_config['mac']
-                    del host_config['mac']
-                    host = ClusterHost(**host_config)
-                    hosts['%s.%s' % (
-                        host_config['hostname'], cluster_name)] = host
-                    host.machine = machines[mac]
-                    host.cluster = clusters[cluster_name]
-                    host.state = HostState(
-                        state="INSTALLING", progress=0.0, message='')
-                    session.add(host)
-
-
 class TestFilterFileExist(unittest2.TestCase):
     def setUp(self):
         super(TestFilterFileExist, self).setUp()
@@ -159,173 +84,24 @@ class TestFileReader(unittest2.TestCase):
     def setUp(self):
         super(TestFileReader, self).setUp()
         logsetting.init()
-        database.create_db()
-        self.config_file = '%s/data/config' % (
-            os.path.dirname(os.path.abspath(__file__)))
 
     def tearDown(self):
         super(TestFileReader, self).tearDown()
-        database.drop_db()
 
-    def test_get_empty_history(self):
-        config_locals = {}
-        config_globals = {}
-        execfile(self.config_file, config_globals, config_locals)
-        prepare_database(config_locals)
-        expected = {
-            'matcher_name': 'start',
-            'progress': 0.0,
-            'message': '',
-            'severity': None
+    def test_readline(self):
+        data = {
+            'pathname': os.path.dirname(
+                os.path.abspath(__file__)) + '/data/sample_log',
+            'log_history': {
+                'position': 0,
+                'partial_line': '',
+            }
         }
-        res = {}
-        reader = file_matcher.FileReader('dummy')
-        history = reader.get_history()
-        res.update(
-            {
-                'matcher_name': history[0],
-                'progress': history[1].progress,
-                'message': history[1].message,
-                'severity': history[1].severity
-            })
-        self.assertEqual(expected, res)
-
-    def test_get_existing_history(self):
-        config_locals = {}
-        config_globals = {}
-        execfile(self.config_file, config_globals, config_locals)
-        prepare_database(config_locals)
-        with database.session() as session:
-            history = LogProgressingHistory(
-                line_matcher_name='start',
-                progress=0.5,
-                message='dummy',
-                severity='INFO',
-                position=0,
-                partial_line='',
-                pathname='dummy')
-            session.add(history)
-
-        expected = {
-            'matcher_name': 'start',
-            'progress': 0.5,
-            'message': 'dummy',
-            'severity': 'INFO'
-        }
-        res = {}
-        reader = file_matcher.FileReader('dummy')
-        history = reader.get_history()
-        res.update({
-            'matcher_name': history[0],
-            'progress': history[1].progress,
-            'message': history[1].message,
-            'severity': history[1].severity
-        })
-        self.assertEqual(expected, res)
-
-    def test_update_history_from_none(self):
-        config_locals = {}
-        config_globals = {}
-        execfile(self.config_file, config_globals, config_locals)
-        prepare_database(config_locals)
-
-        expected = {
-            'progress': 0.5,
-            'line_matcher_name': 'start'
-        }
-        reader = file_matcher.FileReader('dummy')
-        reader.update_history(
-            expected['line_matcher_name'],
-            Progress(
-                progress=expected['progress'],
-                message='',
-                severity='INFO'))
-        res = {}
-        with database.session() as session:
-            history = session.query(
-                LogProgressingHistory).first()
-            res.update({
-                'line_matcher_name': history.line_matcher_name,
-                'progress': history.progress
-            })
-        self.assertEqual(expected, res)
-
-    def test_update_history_from_existing(self):
-        config_locals = {}
-        config_globals = {}
-        execfile(self.config_file, config_globals, config_locals)
-        prepare_database(config_locals)
-
-        with database.session() as session:
-            history = LogProgressingHistory(
-                line_matcher_name='start',
-                progress=0.5,
-                message='dummy',
-                severity='INFO',
-                position=0,
-                partial_line='',
-                pathname='dummy')
-            session.add(history)
-
-        expected = {
-            'progress': 0.8,
-            'line_matcher_name': 'start'
-        }
-        reader = file_matcher.FileReader('dummy')
-        reader.position_ = 1
-        reader.update_history(
-            expected['line_matcher_name'],
-            Progress(
-                progress=expected['progress'],
-                message='',
-                severity='INFO'))
-        res = {}
-        with database.session() as session:
-            history = session.query(
-                LogProgressingHistory).first()
-            res.update({
-                'line_matcher_name': history.line_matcher_name,
-                'progress': history.progress
-            })
-        self.assertEqual(expected, res)
-
-    def test_update_history_failure(self):
-        config_locals = {}
-        config_globals = {}
-        execfile(self.config_file, config_globals, config_locals)
-        prepare_database(config_locals)
-
-        with database.session() as session:
-            history = LogProgressingHistory(
-                line_matcher_name='start',
-                progress=0.5,
-                message='dummy',
-                severity='INFO',
-                position=0,
-                partial_line='',
-                pathname='dummy')
-            session.add(history)
-
-        expected = {
-            'progress': 0.8,
-            'line_matcher_name': 'start'
-        }
-        reader = file_matcher.FileReader('dummy')
-        reader.update_history(
-            expected['line_matcher_name'],
-            Progress(
-                progress=expected['progress'],
-                message='',
-                severity='INFO'))
-        res = {}
-        with database.session() as session:
-            history = session.query(
-                LogProgressingHistory).first()
-            res.update({
-                'line_matcher_name': history.line_matcher_name,
-                'progress': history.progress
-            })
-        self.assertNotEqual(expected, res)
+        matcher = file_matcher.FileReader(**data)
+        lines = list(matcher.readline())
+        expected = ['Line1\n', 'Line2\n', 'Line3\n']
+        for line in lines:
+            self.assertIn(line, expected)
 
 
 class TestFileReaderFactory(unittest2.TestCase):
@@ -339,9 +115,16 @@ class TestFileReaderFactory(unittest2.TestCase):
     def test_get_file_reader_None(self):
         reader_factory = file_matcher.FileReaderFactory(
             'dummy',
-            file_matcher.get_file_filter())
-
-        reader = reader_factory.get_file_reader('dummy', 'dummy')
+        )
+        data = {
+            'hostname': 'dummy',
+            'filename': 'dummy',
+            'log_history': {
+                'position': 0,
+                'partial_line': '',
+            }
+        }
+        reader = reader_factory.get_file_reader(**data)
         self.assertIsNone(reader)
 
 
@@ -353,81 +136,73 @@ class TestFileMatcher(unittest2.TestCase):
     def tearDown(self):
         super(TestFileMatcher, self).tearDown()
 
-    def test_update_absolute_progress_range(self):
-        matcher = file_matcher.FileMatcher(
-            filename='sys.log',
-            min_progress=0.0,
-            max_progress=0.1,
-            line_matchers={
-                'start': LineMatcher(
-                    pattern=r'NOTICE (?P<message>.*)',
-                    progress=IncrementalProgress(.1, .9, .1),
-                    message_template='%(message)s',
+    def test_init_wrong_key(self):
+        data = {
+            'min_progress': 0.0,
+            'max_progress': 0.1,
+            'filename': 'sys.log',
+            'line_matchers': {
+                'dummy': LineMatcher(
+                    pattern=r' ',
                     unmatch_nextline_next_matcher_name='start',
                     match_nextline_next_matcher_name='exit'
-                ),
+                )
             }
+        }
+        self.assertRaises(
+            KeyError,
+            file_matcher.FileMatcher,
+            **data
         )
-        matcher.update_absolute_progress_range(0.5, 1.0)
-        expected = [0.5, 0.55]
-        res = []
-        res.append(matcher.absolute_min_progress_)
-        res.append(matcher.absolute_max_progress_)
-        self.assertEqual(expected, res)
 
-    def test_update_total_progress_none(self):
-        file_progress = Progress(
-            message=None,
-            progress=0.5,
-            severity='info')
-
-        total_progress = file_progress
-        matcher = file_matcher.FileMatcher(
-            filename='sys.log',
-            min_progress=0.0,
-            max_progress=0.1,
-            line_matchers={
+    def test_init_wrong_progress(self):
+        data = {
+            'min_progress': 0.5,
+            'max_progress': 0.1,
+            'filename': 'sys.log',
+            'line_matchers': {
                 'start': LineMatcher(
-                    pattern=r'NOTICE (?P<message>.*)',
-                    progress=IncrementalProgress(.1, .9, .1),
-                    message_template='%(message)s',
+                    pattern=r' ',
                     unmatch_nextline_next_matcher_name='start',
                     match_nextline_next_matcher_name='exit'
-                ),
+                )
             }
+        }
+        self.assertRaises(
+            IndexError,
+            file_matcher.FileMatcher,
+            **data
         )
-        res = matcher.update_total_progress(file_progress, total_progress)
-        self.assertIsNone(res)
 
-    def test_update_total_progress(self):
-        file_progress = Progress(
-            message='dummy',
-            progress=0.5,
-            severity='info')
-
-        total_progress = Progress(
-            message='dummy',
-            progress=0.4,
-            severity='info')
-
-        matcher = file_matcher.FileMatcher(
-            filename='sys.log',
-            min_progress=0.0,
-            max_progress=0.1,
-            line_matchers={
+    def test_update_progress_from_log_history(self):
+        data = {
+            'min_progress': 0.6,
+            'max_progress': 0.9,
+            'filename': 'sys.log',
+            'line_matchers': {
                 'start': LineMatcher(
-                    pattern=r'NOTICE (?P<message>.*)',
-                    progress=IncrementalProgress(.1, .9, .1),
-                    message_template='%(message)s',
+                    pattern=r' ',
                     unmatch_nextline_next_matcher_name='start',
                     match_nextline_next_matcher_name='exit'
-                ),
+                )
             }
+        }
+        matcher = file_matcher.FileMatcher(**data)
+        state = {
+            'message': 'dummy',
+            'severity': 'dummy',
+            'percentage': 0.5
+        }
+        log_history = {
+            'message': 'dummy',
+            'severity': 'dummy',
+            'percentage': 0.7
+        }
+        matcher.update_progress_from_log_history(
+            state,
+            log_history
         )
-        matcher.update_total_progress(file_progress, total_progress)
-        self.assertEqual(
-            0.5,
-            total_progress.progress)
+        self.assertEqual(0.81, state['percentage'])
 
 
 if __name__ == '__main__':

compass/tests/log_analyzor/test_line_matcher.py

@@ -29,27 +29,6 @@ from compass.utils import flags
 from compass.utils import logsetting
 
 
-class TestProgress(unittest2.TestCase):
-    """test class for Progress class."""
-
-    def setUp(self):
-        super(TestProgress, self).setUp()
-        logsetting.init()
-
-    def tearDown(self):
-        super(TestProgress, self).tearDown()
-
-    def test_repr(self):
-        config = {
-            'progress': 0.5,
-            'message': 'dummy',
-            'severity': ''
-        }
-        expected = 'Progress[progress:0.5, message:dummy, severity:]'
-        test_line_matcher = line_matcher.Progress(**config)
-        self.assertEqual(expected, test_line_matcher.__repr__())
-
-
 class TestProgressCalculator(unittest2.TestCase):
     def setUp(self):
         super(TestProgressCalculator, self).setUp()
@@ -60,46 +39,71 @@ class TestProgressCalculator(unittest2.TestCase):
         super(TestProgressCalculator, self).tearDown()
 
     def _mock_progress(self):
-        self.progress = line_matcher.Progress(
-            progress=0.5,
-            message='',
-            severity='')
+        self.log_history = {
+            'percentage': 0.5,
+            'message': '',
+            'severity': ''
+        }
 
     def test_update_progress_progress(self):
         test_1 = {
             'progress_data': 0.7,
             'message': '',
             'severity': '',
-            'progress': self.progress
+            'log_history': self.log_history
         }
         expected_1 = 0.7
         line_matcher.ProgressCalculator.update_progress(
             **test_1)
-        self.assertEqual(expected_1, self.progress.progress)
+        self.assertEqual(expected_1, self.log_history['percentage'])
 
     def test_update_progress_other(self):
         test = {
             'progress_data': 0.5,
             'message': 'dummy',
             'severity': 'dummy',
-            'progress': self.progress
+            'log_history': self.log_history
         }
         expected_message = test['message']
         expected_severity = test['severity']
         line_matcher.ProgressCalculator.update_progress(
             **test)
-        self.assertEqual(expected_message, self.progress.message)
-        self.assertEqual(expected_severity, self.progress.severity)
+        self.assertEqual(expected_message, self.log_history['message'])
+        self.assertEqual(expected_severity, self.log_history['severity'])
 
 
 class TestIncrementalProgress(unittest2.TestCase):
     def setUp(self):
         super(TestIncrementalProgress, self).setUp()
         logsetting.init()
+        self.log_history = {
+            'percentage': 0.5,
+            'message': '',
+            'severity': ''
+        }
 
     def tearDown(self):
         super(TestIncrementalProgress, self).tearDown()
 
+    def test_update(self):
+        test_data = {
+            'min_progress': 0.3,
+            'max_progress': 0.7,
+            'incremental_ratio': 0.5
+        }
+        progress = line_matcher.IncrementalProgress(
+            **test_data)
+        message = 'dummy'
+        severity = 'dummy'
+        log_history = {
+            'percentage': 0.5,
+            'message': '',
+            'severity': ''
+        }
+        expected = 0.7
+        progress.update(message, severity, log_history)
+        self.assertEqual(expected, log_history['percentage'])
+
     def test_init(self):
         test_exceed_one = {
             'min_progress': 1.1,
@@ -131,7 +135,8 @@ class TestIncrementalProgress(unittest2.TestCase):
         self.assertRaises(
             IndexError,
             line_matcher.IncrementalProgress,
-            **test_invalid_ratio)
+            **test_invalid_ratio
+        )
 
 
 class TestRelativeProgress(unittest2.TestCase):
@@ -146,7 +151,8 @@ class TestRelativeProgress(unittest2.TestCase):
         self.assertRaises(
             IndexError,
             line_matcher.RelativeProgress,
-            progress=1.1)
+            progress=1.1
+        )
 
 
 class TestLineMatcher(unittest2.TestCase):
@@ -170,8 +176,13 @@ class TestLineMatcher(unittest2.TestCase):
     def test_regex_not_match(self):
         line = 'abc'
         regex_ = r'^s'
-        progress = line_matcher.Progress(
-            progress=1, message='a', severity=' ')
+        """progress = line_matcher.Progress(
+            progress=1, message='a', severity=' ')"""
+        log_history = {
+            'percentage': 1,
+            'message': 'a',
+            'serverity': ''
+        }
         test_regex_not_match = {
             'pattern': regex_,
             'unmatch_sameline_next_matcher_name': 'usn',
@@ -185,13 +196,16 @@ class TestLineMatcher(unittest2.TestCase):
         self.assertEqual(
             expected,
             matcher.update_progress(
-                line, progress))
+                line, log_history))
 
     def test_regex_match(self):
         line = 'abc'
         regex_ = r'^a'
-        progress = line_matcher.Progress(
-            progress=1, message='a', severity=' ')
+        log_history = {
+            'percentage': 1,
+            'message': 'a',
+            'serverity': ''
+        }
         test_regex_match = {
             'pattern': regex_,
             'unmatch_sameline_next_matcher_name': 'usn',
@@ -205,12 +219,16 @@ class TestLineMatcher(unittest2.TestCase):
         self.assertEqual(
             expected,
             matcher.update_progress(
-                line, progress))
+                line, log_history)
+        )
 
     def test_wrong_message(self):
         line = 'abc'
-        progress = line_matcher.Progress(
-            progress=1, message='a', severity=' ')
+        log_history = {
+            'percentage': 1,
+            'message': 'a',
+            'serverity': ''
+        }
         test_wrong_message = {
             'pattern': r'.*.',
             'message_template': 'Installing %(package)s'
@@ -221,7 +239,8 @@ class TestLineMatcher(unittest2.TestCase):
             KeyError,
             matcher.update_progress,
             line=line,
-            progress=progress)
+            log_history=log_history
+        )
 
 if __name__ == '__main__':
     flags.init()