Split validation at 3 layers

We can group validation by 3 types: syntax (check that config is valid
for plugin in terms of jsonschema, required arguments, etc),
platform (check that deployment has credentials for launching
the plugin on specific environment), semantic(all others which operates
users: for example validate that specific service is enabled).

Splitting validation by these groups is required to support "validators
for validators". I mean that if we check that syntax of config is valid
and that we have credentials for all required platforms, we able to
execute more complex validators and ensure that they will not failed on
"random" failures like "AttributeError", "KeyError" and etc.

Change-Id: I1ea38403af6cfbd6b7765ae3c0caca12759bc571

rally/plugins/openstack/context/keystone/users.py

@@ -30,6 +30,7 @@ from rally.plugins.openstack.services.identity import identity
 from rally.plugins.openstack.wrappers import network
 from rally.task import context
 from rally.task import utils
+from rally.task import validation
 
 from rally.common import opts
 opts.register()
@@ -45,6 +46,7 @@ PROJECT_DOMAIN_DESCR = "ID of domain in which projects will be created."
 USER_DOMAIN_DESCR = "ID of domain in which users will be created."
 
 
+@validation.add("required_platform", platform="openstack", admin=True)
 @context.configure(name="users", namespace="openstack", order=100)
 class UserGenerator(context.Context):
     """Context class for generating temporary users/tenants for benchmarks."""

tests/unit/doc/test_task_samples.py

@@ -40,10 +40,7 @@ class TaskSampleTestCase(test.TestCase):
         if os.environ.get("TOX_ENV_NAME") == "cover":
             self.skipTest("There is no need to check samples in coverage job.")
 
-    @mock.patch("rally.task.engine.TaskEngine"
-                "._validate_config_semantic")
-    def test_schema_is_valid(self,
-                             mock_task_engine__validate_config_semantic):
+    def test_schema_is_valid(self):
         scenarios = set()
 
         for dirname, dirnames, filenames in os.walk(self.samples_path):
@@ -61,7 +58,7 @@ class TaskSampleTestCase(test.TestCase):
                                                      (task_file.read()))
                         eng = engine.TaskEngine(task_config,
                                                 mock.MagicMock(), mock.Mock())
-                        eng.validate()
+                        eng.validate(only_syntax=True)
                     except Exception:
                         print(traceback.format_exc())
                         self.fail("Invalid task file: %s" % full_path)

tests/unit/rally_jobs/test_jobs.py

@@ -29,10 +29,7 @@ class RallyJobsTestCase(test.TestCase):
     rally_jobs_path = os.path.join(
         os.path.dirname(rally.__file__), "..", "rally-jobs")
 
-    @mock.patch("rally.task.engine.TaskEngine"
-                "._validate_config_semantic")
-    def test_schema_is_valid(
-            self, mock_task_engine__validate_config_semantic):
+    def test_schema_is_valid(self):
         discover.load_plugins(os.path.join(self.rally_jobs_path, "plugins"))
 
         files = {f for f in os.listdir(self.rally_jobs_path)
@@ -64,7 +61,7 @@ class RallyJobsTestCase(test.TestCase):
 
                     eng = engine.TaskEngine(task, mock.MagicMock(),
                                             mock.Mock())
-                    eng.validate()
+                    eng.validate(only_syntax=True)
                 except Exception:
                     print(traceback.format_exc())
                     self.fail("Wrong task input file: %s" % full_path)