Expose skip list mechanism via the CLI

The skip list feature was available only via the API, this patch
allow the user to pass a file with the list of the validations
which he wants to skip during the run, example:

check-ram: hosts: all
  reason: Wrong ram value
  lp: https://lp.fake.net
check-cpu: hosts: undercloud
  reason: Unstable validation
  lp: https://lp.fake.net
Change-Id: I04059f9339085e0dcef1f018cad1be511ee7d3c7

README.rst

@@ -59,4 +59,31 @@ Then run validations::
 
     validation.py run --validation check-ftype,512e --inventory /etc/ansible/hosts
 
+
+Skip list
+=========
+
+You can provide a file with a list of Validations to skip via the run command::
+
+    validation.py run --validation check-ftype,512e --inventory /etc/ansible/hosts --skiplist my-skip-list.yaml
+
+This file should be formed as::
+
+    validation-name:
+      hosts: targeted_hostname
+      reason: reason to ignore the file
+      lp: bug number
+
+The framework will skip the validation against the ``hosts`` key.
+In order to skip the validation on every hosts, you can set ``all`` value such
+as::
+
+    hosts: all
+
+If no hosts key is provided for a given validation, it will be considered as ``hosts: all``.
+
+.. note::
+    The ``reason`` and ``lp`` key are for tracking and documentation purposes,
+    the framework won't use those keys.
+
 .. _Apache_license: http://www.apache.org/licenses/LICENSE-2.0

skiplist-example.yaml

@@ -0,0 +1,11 @@
+check-ram:
+  hosts: all
+  # reason and lp key is not mandatory for the VF. Those values are in the list
+  # in order to track the reason and eventually the related bug number of the
+  # skipped validation.
+  reason: Wrong ram value
+  lp: https://lp.fake.net
+check-cpu:
+  hosts: undercloud
+  reason: Unstable validation
+  lp: https://lp.fake.net

validations_libs/cli/common.py

@@ -102,16 +102,16 @@ def write_junitxml(output_junitxml, results):
         output.write(to_xml_report_string([ts]))
 
 
-def read_extra_vars_file(extra_vars_file):
+def read_cli_data_file(data_file):
-    """Read file containing extra variables.
+    """Read CLI data (YAML/JSON) file.
     """
     try:
-        with open(extra_vars_file, 'r') as env_file:
+        with open(data_file, 'r') as _file:
-            return yaml.safe_load(env_file.read())
+            return yaml.safe_load(_file.read())
-    except yaml.YAMLError as error:
+    except (yaml.YAMLError, IOError) as error:
         error_msg = (
-            "The extra_vars file must be properly formatted YAML/JSON."
+            "The file {} must be properly formatted YAML/JSON."
-            "Details: {}.").format(error)
+            "Details: {}.").format(data_file, error)
         raise RuntimeError(error_msg)
 
 

validations_libs/cli/run.py

@@ -97,6 +97,13 @@ class Run(BaseCommand):
                 "KEY multiple times, the last given VALUE for that same KEY "
                 "will override the other(s)"))
 
+        parser.add_argument('--skiplist', dest='skip_list',
+                            default=None,
+                            help=("Path where the skip list is stored. "
+                                  "An example of the skiplist format could "
+                                  "be found at the root of the "
+                                  "validations-libs repository."))
+
         extra_vars_group = parser.add_mutually_exclusive_group(required=False)
         extra_vars_group.add_argument(
             '--extra-vars',
@@ -184,9 +191,15 @@ class Run(BaseCommand):
                 "Loading extra vars file {}".format(
                     parsed_args.extra_vars_file))
 
-            extra_vars = common.read_extra_vars_file(
+            extra_vars = common.read_cli_data_file(
                 parsed_args.extra_vars_file)
 
+        skip_list = None
+        if parsed_args.skip_list:
+            skip_list = common.read_cli_data_file(parsed_args.skip_list)
+            if not isinstance(skip_list, dict):
+                raise RuntimeError("Wrong format for the skiplist.")
+
         try:
             results = v_actions.run_validations(
                 inventory=parsed_args.inventory,
@@ -203,7 +216,8 @@ class Run(BaseCommand):
                 quiet=quiet_mode,
                 ssh_user=parsed_args.ssh_user,
                 log_path=parsed_args.validation_log_dir,
-                validation_config=config)
+                validation_config=config,
+                skip_list=skip_list)
         except RuntimeError as e:
             raise RuntimeError(e)
 

validations_libs/tests/cli/test_common.py

@@ -0,0 +1,47 @@
+#   Copyright 2021 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+from unittest import TestCase
+import yaml
+
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+from validations_libs.cli import common
+
+
+class TestCommon(TestCase):
+
+    def setUp(self):
+        super(TestCommon, self).setUp()
+
+    def test_read_cli_data_file_with_example_file(self):
+        example_data = {'check-cpu': {'hosts': 'undercloud',
+                                      'lp': 'https://lp.fake.net',
+                                      'reason': 'Unstable validation'},
+                        'check-ram': {'hosts': 'all',
+                                      'lp': 'https://lp.fake.net',
+                                      'reason': 'Wrong ram value'}}
+        data = common.read_cli_data_file('skiplist-example.yaml')
+        self.assertEqual(data, example_data)
+
+    @mock.patch('six.moves.builtins.open', side_effect=IOError)
+    def test_read_cli_data_file_ioerror(self, mock_open):
+        self.assertRaises(RuntimeError, common.read_cli_data_file, 'foo')
+
+    @mock.patch('yaml.safe_load', side_effect=yaml.YAMLError)
+    def test_read_cli_data_file_yaml_error(self, mock_yaml):
+        self.assertRaises(RuntimeError, common.read_cli_data_file, 'foo')

validations_libs/tests/cli/test_run.py

@@ -83,7 +83,8 @@ class TestRun(BaseCommand):
             'quiet': True,
             'ssh_user': 'doe',
             'log_path': mock_log_dir,
-            'validation_config': {}
+            'validation_config': {},
+            'skip_list': None
             }
 
         arglist = ['--validation', 'foo',
@@ -121,7 +122,8 @@ class TestRun(BaseCommand):
             'quiet': True,
             'ssh_user': 'doe',
             'log_path': mock_log_dir,
-            'validation_config': {}
+            'validation_config': {},
+            'skip_list': None
             }
 
         arglist = ['--validation', 'foo',
@@ -172,7 +174,8 @@ class TestRun(BaseCommand):
             'quiet': True,
             'ssh_user': 'doe',
             'log_path': mock_log_dir,
-            'validation_config': {}
+            'validation_config': {},
+            'skip_list': None
             }
 
         arglist = ['--validation', 'foo',
@@ -208,7 +211,8 @@ class TestRun(BaseCommand):
             'quiet': True,
             'ssh_user': 'doe',
             'log_path': mock_log_dir,
-            'validation_config': {}
+            'validation_config': {},
+            'skip_list': None
             }
 
         arglist = ['--validation', 'foo',
@@ -248,7 +252,8 @@ class TestRun(BaseCommand):
             'python_interpreter': sys.executable,
             'quiet': False,
             'ssh_user': 'doe',
-            'validation_config': {}
+            'validation_config': {},
+            'skip_list': None
             }
 
         arglist = ['--validation', 'foo',
@@ -285,7 +290,8 @@ class TestRun(BaseCommand):
             'quiet': True,
             'ssh_user': 'doe',
             'log_path': mock_log_dir,
-            'validation_config': {}
+            'validation_config': {},
+            'skip_list': None
             }
 
         arglist = ['--validation', 'foo',
@@ -325,7 +331,8 @@ class TestRun(BaseCommand):
             'quiet': True,
             'ssh_user': 'doe',
             'log_path': mock_log_dir,
-            'validation_config': {}
+            'validation_config': {},
+            'skip_list': None
             }
 
         arglist = ['--validation', 'foo',
@@ -372,7 +379,8 @@ class TestRun(BaseCommand):
             'quiet': True,
             'ssh_user': 'doe',
             'log_path': mock_log_dir,
-            'validation_config': {}
+            'validation_config': {},
+            'skip_list': None
             }
 
         arglist = ['--validation', 'foo']
@@ -405,7 +413,8 @@ class TestRun(BaseCommand):
             'python_interpreter': sys.executable,
             'quiet': True,
             'ssh_user': 'doe',
-            'validation_config': {}
+            'validation_config': {},
+            'skip_list': None
             }
 
         arglist = ['--validation', 'foo']
@@ -442,7 +451,8 @@ class TestRun(BaseCommand):
             'quiet': True,
             'ssh_user': 'doe',
             'log_path': mock_log_dir,
-            'validation_config': {}
+            'validation_config': {},
+            'skip_list': None
             }
 
         self._set_args(arglist)
@@ -479,10 +489,73 @@ class TestRun(BaseCommand):
             'quiet': True,
             'ssh_user': 'doe',
             'log_path': mock_log_dir,
-            'validation_config': {}
+            'validation_config': {},
+            'skip_list': None
             }
 
         self._set_args(arglist)
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
         self.cmd.take_action(parsed_args)
         mock_run.assert_called_with(**run_called_args)
+
+    @mock.patch('validations_libs.constants.VALIDATIONS_LOG_BASEDIR')
+    @mock.patch('yaml.safe_load', return_value={'key': 'value'})
+    @mock.patch('six.moves.builtins.open')
+    @mock.patch('getpass.getuser',
+                return_value='doe')
+    @mock.patch('validations_libs.validation_actions.ValidationActions.'
+                'run_validations',
+                return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN))
+    @mock.patch('validations_libs.utils.load_config', return_value={})
+    def test_run_command_with_skip_list(self, mock_config, mock_run,
+                                        mock_user, mock_open,
+                                        mock_yaml, mock_log_dir):
+
+        run_called_args = {
+            'inventory': 'localhost',
+            'limit_hosts': None,
+            'group': [],
+            'category': [],
+            'product': [],
+            'extra_vars': None,
+            'validations_dir': '/usr/share/ansible/validation-playbooks',
+            'base_dir': '/usr/share/ansible',
+            'validation_name': ['foo'],
+            'extra_env_vars': None,
+            'python_interpreter': sys.executable,
+            'quiet': True,
+            'ssh_user': 'doe',
+            'log_path': mock_log_dir,
+            'validation_config': {},
+            'skip_list': {'key': 'value'}
+            }
+
+        arglist = ['--validation', 'foo',
+                   '--skiplist', '/foo/skip.yaml']
+        verifylist = [('validation_name', ['foo']),
+                      ('skip_list', '/foo/skip.yaml')]
+        self._set_args(arglist)
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+        self.cmd.take_action(parsed_args)
+        mock_run.assert_called_with(**run_called_args)
+
+    @mock.patch('validations_libs.constants.VALIDATIONS_LOG_BASEDIR')
+    @mock.patch('yaml.safe_load', return_value=[{'key': 'value'}])
+    @mock.patch('six.moves.builtins.open')
+    @mock.patch('getpass.getuser',
+                return_value='doe')
+    @mock.patch('validations_libs.validation_actions.ValidationActions.'
+                'run_validations',
+                return_value=copy.deepcopy(fakes.FAKE_SUCCESS_RUN))
+    @mock.patch('validations_libs.utils.load_config', return_value={})
+    def test_run_command_with_skip_list_bad_format(self, mock_config, mock_run,
+                                                   mock_user, mock_open,
+                                                   mock_yaml, mock_log_dir):
+
+        arglist = ['--validation', 'foo',
+                   '--skiplist', '/foo/skip.yaml']
+        verifylist = [('validation_name', ['foo']),
+                      ('skip_list', '/foo/skip.yaml')]
+        self._set_args(arglist)
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+        self.assertRaises(RuntimeError, self.cmd.take_action, parsed_args)

validations_libs/validation_actions.py

@@ -188,8 +188,8 @@ class ValidationActions(object):
             'cloud2,!cloud1'
 
         """
-        hosts = skip_list[playbook].get('hosts')
+        hosts = skip_list[playbook].get('hosts', 'all')
-        if hosts == 'ALL' or hosts is None:
+        if hosts.lower() == 'all':
             return None
         else:
             _hosts = ['!{}'.format(hosts)]
@@ -464,8 +464,17 @@ class ValidationActions(object):
                                 'validations': _playbook.split('.')[0],
                                 'UUID': validation_uuid,
                                 })
+                # Print hosts which has been skipped:
+                if _hosts:
+                    skipped_hosts = [h.replace('!', '')
+                                     for h in _hosts.split(',') if '!' in h]
+                    if skipped_hosts:
+                        msg = ("Validation {} has been skipped "
+                               "on hosts: {}").format(_play,
+                                                      ','.join(skipped_hosts))
+                        self.log.info(msg)
             else:
-                self.log.debug('Skipping Validations: {}'.format(playbook))
+                self.log.info('Skipping Validations: {}'.format(playbook))
 
         if run_async:
             return results