Merge "[samples] Speed up checking samples"

samples/tasks/scenarios/glance/create-and-deactivate-image.json

@@ -16,6 +16,11 @@
                     "tenants": 1,
                     "users_per_tenant": 1
                 }
+            },
+            "sla": {
+                "failure_rate": {
+                    "max": 0
+                }
             }
         }
     ]

samples/tasks/scenarios/glance/create-and-deactivate-image.yaml

@@ -13,3 +13,6 @@
         users:
           tenants: 1
           users_per_tenant: 1
+      sla:
+        failure_rate:
+          max: 0

samples/tasks/scenarios/sahara/jobs/dfsio-job-sequence-scaling.json

@@ -55,6 +55,11 @@
                     "auto_security_group": true
                 },
                 "network": {}
+            },
+            "sla": {
+                "failure_rate": {
+                    "max": 0
+                }
             }
         }
     ]

samples/tasks/scenarios/sahara/jobs/dfsio-job-sequence-scaling.yaml

@@ -54,3 +54,6 @@
           hadoop_version: "2.6.0"
           auto_security_group: True
         network: {}
+      sla:
+        failure_rate:
+          max: 0

samples/tasks/scenarios/sahara/jobs/dfsio-job-sequence.json

@@ -55,6 +55,11 @@
                     "auto_security_group": true
                 },
                 "network": {}
+            },
+            "sla": {
+                "failure_rate": {
+                    "max": 0
+                }
             }
         }
     ]

samples/tasks/scenarios/sahara/jobs/dfsio-job-sequence.yaml

@@ -51,3 +51,6 @@
           hadoop_version: "2.6.0"
           auto_security_group: True
         network: {}
+      sla:
+        failure_rate:
+          max: 0

samples/tasks/scenarios/sahara/jobs/java-action-job.json

@@ -43,6 +43,11 @@
                     "auto_security_group": true
                 },
                 "network": {}
+            },
+            "sla": {
+                "failure_rate": {
+                    "max": 0
+                }
             }
         }
     ]

samples/tasks/scenarios/sahara/jobs/java-action-job.yaml

@@ -35,3 +35,6 @@
           hadoop_version: "2.6.0"
           auto_security_group: True
         network: {}
+      sla:
+        failure_rate:
+          max: 0

samples/tasks/scenarios/sahara/jobs/pig-script-job.json

@@ -52,6 +52,11 @@
                     "auto_security_group": true
                 },
                 "network": {}
+            },
+            "sla": {
+                "failure_rate": {
+                    "max": 0
+                }
             }
         }
     ]

samples/tasks/scenarios/sahara/jobs/pig-script-job.yaml

@@ -40,3 +40,6 @@
           hadoop_version: "2.6.0"
           auto_security_group: True
         network: {}
+      sla:
+        failure_rate:
+          max: 0

tests/unit/doc/test_task_samples.py

@@ -14,9 +14,8 @@
 #    under the License.
 
 import inspect
-import json
+import itertools
 import os
-import re
 import traceback
 
 import mock
@@ -29,7 +28,7 @@ from rally.task import engine
 from rally.task import scenario
 from tests.unit import test
 
-RALLY_PATH = os.path.join(os.path.dirname(rally.__file__), os.pardir)
+RALLY_PATH = os.path.dirname(os.path.dirname(rally.__file__))
 
 
 class TaskSampleTestCase(test.TestCase):
@@ -42,51 +41,56 @@ class TaskSampleTestCase(test.TestCase):
         with mock.patch("rally.api.API.check_db_revision"):
             self.rapi = api.API()
 
-    def _check_missing_sla_section(self, raw, filename):
+    def iterate_samples(self, merge_pairs=True):
-        re_sla_section = re.compile(r"\"?failure_rate\"?:")
+        """Iterates all task samples
-        matches = re_sla_section.findall(raw)
-        self.assertTrue(
-            len(matches),
-            "sla section is required in %s." % filename)
 
-    def test_check_missing_sla_section(self):
+        :param merge_pairs: Whether or not to return both json and yaml samples
+            of one sample.
+        """
         for dirname, dirnames, filenames in os.walk(self.samples_path):
             for filename in filenames:
-                full_path = os.path.join(dirname, filename)
+                # NOTE(hughsaunders): Skip non config files
+                # (bug https://bugs.launchpad.net/rally/+bug/1314369)
+                if filename.endswith("json") or (
+                        not merge_pairs and filename.endswith("yaml")):
+                    yield os.path.join(dirname, filename)
 
-                if not full_path.startswith("./samples/tasks/scenarios/"):
+    def test_check_missing_sla_section(self):
+        failures = []
+        for path in self.iterate_samples():
+            if "tasks/scenarios" not in path:
                 continue
-
+            with open(path) as task_file:
-                if not re.search("\.(ya?ml|json)$", filename, flags=re.I):
+                task_config = yaml.safe_load(
-                    continue
+                    self.rapi.task.render_template(
-
+                        task_template=task_file.read()))
-                with open(full_path) as task_file:
+                for workload in itertools.chain(*task_config.values()):
-                    data = task_file.read()
+                    if not workload.get("sla", {}):
-                    self._check_missing_sla_section(data, task_file)
+                        failures.append(path)
+        if failures:
+            self.fail("One or several workloads from the list of samples below"
+                      " doesn't have SLA section: \n  %s" %
+                      "\n  ".join(failures))
 
     def test_schema_is_valid(self):
         scenarios = set()
 
-        for dirname, dirnames, filenames in os.walk(self.samples_path):
+        for path in self.iterate_samples():
-            for filename in filenames:
+            with open(path) as task_file:
-                full_path = os.path.join(dirname, filename)
+                try:
-
-                # NOTE(hughsaunders): Skip non config files
-                # (bug https://bugs.launchpad.net/rally/+bug/1314369)
-                if not re.search("\.(ya?ml|json)$", filename, flags=re.I):
-                    continue
-
-                with open(full_path) as task_file:
                     try:
                         task_config = yaml.safe_load(
                             self.rapi.task.render_template(
                                 task_template=task_file.read()))
+                    except Exception:
+                        print(traceback.format_exc())
+                        self.fail("Invalid JSON file: %s" % path)
                     eng = engine.TaskEngine(task_config,
                                             mock.MagicMock(), mock.Mock())
                     eng.validate(only_syntax=True)
                 except Exception:
                     print(traceback.format_exc())
-                        self.fail("Invalid task file: %s" % full_path)
+                    self.fail("Invalid task file: %s" % path)
                 else:
                     scenarios.update(task_config.keys())
 
@@ -98,54 +102,30 @@ class TaskSampleTestCase(test.TestCase):
         self.assertEqual(missing, [],
                          "These scenarios don't have samples: %s" % missing)
 
-    def test_json_correct_syntax(self):
+    def test_task_config_pairs(self):
-        for dirname, dirnames, filenames in os.walk(self.samples_path):
+
-            for filename in filenames:
+        not_equal = []
-                if not filename.endswith(".json"):
+        missed = []
+        checked = []
+
+        for path in self.iterate_samples(merge_pairs=False):
+            if path.endswith(".json"):
+                json_path = path
+                yaml_path = json_path.replace(".json", ".yaml")
+            else:
+                yaml_path = path
+                json_path = yaml_path.replace(".yaml", ".json")
+
+            if json_path in checked:
                 continue
-                full_path = os.path.join(dirname, filename)
+            else:
-                with open(full_path) as task_file:
+                checked.append(json_path)
-                    try:
-                        json.loads(self.rapi.task.render_template(
-                            task_template=task_file.read()))
-                    except Exception:
-                        print(traceback.format_exc())
-                        self.fail("Invalid JSON file: %s" % full_path)
-
-    def test_task_config_pair_existance(self):
-        inexistent_paths = []
-
-        for dirname, dirnames, filenames in os.walk(self.samples_path):
-            # iterate over unique config names
-            for sample_name in set(
-                    f[:-5] for f in filenames
-                    if f.endswith(".json") or f.endswith(".yaml")):
-
-                partial_path = os.path.join(dirname, sample_name)
-                yaml_path = partial_path + ".yaml"
-                json_path = partial_path + ".json"
 
             if not os.path.exists(yaml_path):
-                    inexistent_paths.append(yaml_path)
+                missed.append(yaml_path)
             elif not os.path.exists(json_path):
-                    inexistent_paths.append(json_path)
+                missed.append(json_path)
-
+            else:
-        if inexistent_paths:
-            self.fail("Sample task configs are missing:\n%r"
-                      % inexistent_paths)
-
-    def test_task_config_pairs_equality(self):
-        for dirname, dirnames, filenames in os.walk(self.samples_path):
-            # iterate over unique config names
-            for sample_name in set(
-                    f[:-5] for f in filenames
-                    if f.endswith(".json") or f.endswith(".yaml")):
-
-                partial_path = os.path.join(dirname, sample_name)
-                yaml_path = partial_path + ".yaml"
-                json_path = partial_path + ".json"
-
-                if os.path.exists(yaml_path) and os.path.exists(json_path):
                 with open(json_path) as json_file:
                     json_config = yaml.safe_load(
                         self.rapi.task.render_template(
@@ -154,24 +134,31 @@ class TaskSampleTestCase(test.TestCase):
                     yaml_config = yaml.safe_load(
                         self.rapi.task.render_template(
                             task_template=yaml_file.read()))
-                    self.assertEqual(json_config, yaml_config,
+                if json_config != yaml_config:
-                                     "Sample task configs are not equal:"
+                    not_equal.append("'%s' and '%s'" % (yaml_path, json_path))
-                                     "\n%s\n%s" % (yaml_path, json_path))
+
+        error = ""
+        if not_equal:
+            error += ("Sample task configs are not equal:\n\t%s\n"
+                      % "\n\t".join(not_equal))
+        if missed:
+            self.fail("Sample task configs are missing:\n\t%s\n"
+                      % "\n\t".join(missed))
+
+        if error:
+            self.fail(error)
 
     def test_no_underscores_in_filename(self):
         bad_filenames = []
 
-        for dirname, dirnames, filenames in os.walk(self.samples_path):
+        for path in self.iterate_samples(merge_pairs=False):
-            for filename in filenames:
+            if "_" in path:
-                if "_" in filename and (filename.endswith(".yaml") or
+                bad_filenames.append(path)
-                                        filename.endswith(".json")):
-                    full_path = os.path.join(dirname, filename)
-                    bad_filenames.append(full_path)
 
-        self.assertEqual([], bad_filenames,
+        if bad_filenames:
-                         "Following sample task filenames contain "
+            self.fail("Following sample task filename(s) contain underscores "
-                         "underscores (_) but must use dashes (-) instead: "
+                      "(_) but must use dashes (-) instead: %s" %
-                         "{}".format(bad_filenames))
+                      bad_filenames)
 
     def test_context_samples_found(self):
         all_plugins = context.Context.get_all()