Merge "make verification job great again"

tests/ci/rally_verify.py

@@ -13,6 +13,7 @@
 #    under the License.
 
 import argparse
+import collections
 import gzip
 import json
 import logging
@@ -22,160 +23,151 @@ import subprocess
 import sys
 import uuid
 
-from rally.cli import envutils
+from rally import api
-from rally.plugins.openstack import credential
 from rally.ui import utils
 
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.DEBUG)
 
-BASE_DIR = "rally-verify"
-
-MODES = {"full": "--pattern set=full", "light": "--pattern set=smoke"}
-DEPLOYMENT_NAME = "devstack"
-VERIFIER_TYPE = "tempest"
-VERIFIER_SOURCE = "https://git.openstack.org/openstack/tempest"
-VERIFIER_EXT_REPO = "https://git.openstack.org/openstack/keystone"
-VERIFIER_EXT_NAME = "keystone_tests"
-SKIP_TESTS = (
-    "tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON."
-    "test_get_flavor[id-1f12046b-753d-40d2-abb6-d8eb8b30cb2f,smoke]: "
-    "This test was skipped intentionally")
-XFAIL_TESTS = (
-    "tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON."
-    "test_get_vnc_console[id-c6bc11bf-592e-4015-9319-1c98dc64daf5]: "
-    "This test fails because 'novnc' console type is unavailable")
-TEST_NAME_RE = re.compile(r"^[a-zA-Z_.0-9]+(\[[a-zA-Z-,=0-9]*\])?$")
-
 # NOTE(andreykurilin): this variable is used to generate output file names
 # with prefix ${CALL_COUNT}_ .
 _call_count = 0
-# NOTE(andreykurilin): if some command fails, script should end with
-# error status
-_return_status = 0
 
 
-def call_rally(cmd, print_output=False, output_type=None):
+class Status(object):
-    """Execute a Rally command and write result in files."""
+    PASS = "success"
-    global _return_status
+    ERROR = "error"
+    SKIPPED = "skip"
+    FAILURE = "fail"
+
+
+class Step(object):
+    COMMAND = None
+    DEPENDS_ON = None
+    CALL_ARGS = {}
+
+    BASE_DIR = "rally-verify"
+    HTML_TEMPLATE = ("<span class=\"%(status)s\">[%(status)s]</span>\n"
+                     "<a href=\"%(output_file)s\">%(doc)s</a>\n"
+                     "<code>$ %(cmd)s</code>")
+
+    def __init__(self, args, rapi):
+        self.args = args
+        self.rapi = rapi
+        self.result = {"status": Status.PASS,
+                       "doc": self.__doc__}
+
+    @property
+    def name(self):
+        return " ".join(re.findall("[A-Z][^A-Z]*",
+                                   self.__class__.__name__)).lower()
+
+    def check(self, results):
+        """Check weather this step should be executed or skipped."""
+        if self.DEPENDS_ON is not None:
+            if results[self.DEPENDS_ON].result["status"] in (
+                    Status.PASS, Status.FAILURE):
+                return True
+            else:
+                self.result["status"] = Status.SKIPPED
+                msg = ("Step '%s' is skipped, since depends on step '%s' is "
+                       "skipped or finished with an error." %
+                       (self.name, results[self.DEPENDS_ON].name))
+                stdout_file = self._generate_path(
+                    "%s.txt.gz" % self.__class__.__name__)
+
+                self.result["output_file"] = self._write_file(
+                    stdout_file, msg, compress=True)
+                return False
+        return True
+
+    def setUp(self):
+        """Obtain variables required for execution"""
+        pass
+
+    def run(self):
+        """Execute step. The default action - execute the command"""
+        self.setUp()
+
+        cmd = "rally --rally-debug verify %s" % (self.COMMAND % self.CALL_ARGS)
+        self.result["cmd"] = cmd
+        self.result["status"], self.result["output"] = self.call_rally(cmd)
+
+        stdout_file = self._generate_path("%s.txt.gz" % cmd)
+        self.result["output_file"] = self._write_file(
+            stdout_file, self.result["output"], compress=True)
+
+    @classmethod
+    def _generate_path(cls, root):
         global _call_count
         _call_count += 1
 
-    data = {"cmd": "rally --rally-debug %s" % cmd}
+        root = root.replace("<", "").replace(">", "").replace("/", "_")
-    stdout_file = "{base_dir}/{prefix}_{cmd}.txt.gz"
+        parts = ["%s" % _call_count]
+        for path in root.split(" "):
+            if path.startswith(cls.BASE_DIR):
+                path = path[len(cls.BASE_DIR) + 1:]
+            parts.append(path)
+        return os.path.join(cls.BASE_DIR, "_".join(parts))
 
-    cmd = cmd.replace("/", "_")
+    @classmethod
-    data.update({"stdout_file": stdout_file.format(base_dir=BASE_DIR,
+    def _write_file(cls, path, data, compress=False):
-                                                   prefix=_call_count,
-                                                   cmd=cmd.replace(" ", "_"))})
-
-    if output_type:
-        data["output_file"] = data["stdout_file"].replace(
-            ".txt.", ".%s." % output_type)
-        data["cmd"] += " --to %s" % data["output_file"]
-        data["cmd"] += " --type %s" % output_type
-
-    try:
-        LOG.info("Try to execute `%s`." % data["cmd"])
-        stdout = subprocess.check_output(data["cmd"].split(),
-                                         stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError as e:
-        LOG.error("Command `%s` failed." % data["cmd"])
-        stdout = e.output
-        data["status"] = "fail"
-        _return_status = 1
-    else:
-        data["status"] = "success"
-
-    if output_type:
-        # let's gzip results
-        with open(data["output_file"]) as f:
-            output = f.read()
-        with gzip.open(data["output_file"], "wb") as f:
-            f.write(output)
-
-    stdout = "$ %s\n%s" % (data["cmd"], stdout)
-
-    with gzip.open(data["stdout_file"], "wb") as f:
-        f.write(stdout)
-
-    if print_output:
-        print(stdout)
-
-    return data
-
-
-def start_verification(args):
-    """Start a verification, show results and generate reports."""
-    results = call_rally("verify start %s" % args)
-    results["uuid"] = envutils.get_global(envutils.ENV_VERIFICATION)
-    results["show"] = call_rally("verify show")
-    results["show_detailed"] = call_rally("verify show --detailed")
-    for output_type in ("json", "html", "junit-xml"):
-        results[output_type.replace("-", "_")] = call_rally(
-            "verify report", output_type=output_type)
-    # NOTE(andreykurilin): we need to clean verification uuid from global
-    # environment to be able to load it next time(for another verification).
-    envutils.clear_global(envutils.ENV_VERIFICATION)
-    return results
-
-
-def write_file(filename, data):
         """Create a file and write some data to it."""
-    path = os.path.join(BASE_DIR, filename)
+        if compress:
+            with gzip.open(path, "wb") as f:
+                f.write(data)
+        else:
             with open(path, "wb") as f:
                 f.write(data)
         return path
 
+    @staticmethod
+    def call_rally(command):
+        """Execute a Rally verify command."""
+        try:
+            LOG.info("Try to execute `%s`." % command)
+            stdout = subprocess.check_output(command.split(),
+                                             stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError as e:
+            LOG.error("Command `%s` failed." % command)
+            return Status.ERROR, e.output
+        else:
+            return Status.PASS, stdout
 
-def generate_trends_reports(uuid_1, uuid_2):
+    def to_html(self):
-    """Generate trends reports."""
+        return self.HTML_TEMPLATE % self.result
-    results = {}
-    for output_type in ("json", "html", "junit-xml"):
-        results[output_type.replace("-", "_")] = call_rally(
-            "verify report --uuid %s %s" % (uuid_1, uuid_2),
-            output_type=output_type)
-    return results
 
 
-def render_page(**render_vars):
+class SetUpStep(Step):
-    template = utils.get_template("ci/index_verify.html")
+    """Validate deployment, create required resources and directories."""
-    with open(os.path.join(BASE_DIR, "extra/index.html"), "w") as f:
-        f.write(template.render(**render_vars))
 
+    DEPLOYMENT_NAME = "devstack"
 
-def main():
+    def run(self):
-    parser = argparse.ArgumentParser(description="Launch rally-verify job.")
+        if not os.path.exists("%s/extra" % self.BASE_DIR):
-    parser.add_argument("--mode", type=str, default="light",
+            os.makedirs("%s/extra" % self.BASE_DIR)
-                        help="Mode of job. The 'full' mode corresponds to the "
-                             "full set of verifier tests. The 'light' mode "
-                             "corresponds to the smoke set of verifier tests.",
-                        choices=MODES.keys())
-    parser.add_argument("--compare", action="store_true",
-                        help="Start the second verification to generate a "
-                             "trends report for two verifications.")
-    # TODO(ylobankov): Remove hard-coded Tempest related things and make it
-    #                  configurable.
-    parser.add_argument("--ctx-create-resources", action="store_true",
-                        help="Make Tempest context create needed resources "
-                             "for the tests.")
 
-    args = parser.parse_args()
+        # ensure that deployment exit
+        deployment = self.rapi.deployment._get(self.DEPLOYMENT_NAME)
+        # check it
+        result = self.rapi.deployment.check(
+            deployment=self.DEPLOYMENT_NAME)["openstack"]
+        if "admin_error" in result[0] or "user_error" in result[0]:
+            self.result["status"] = Status.ERROR
+            return
 
-    if not os.path.exists("%s/extra" % BASE_DIR):
+        try:
-        os.makedirs("%s/extra" % BASE_DIR)
+            subprocess.check_call(["rally", "deployment", "use",
+                                   "--deployment", self.DEPLOYMENT_NAME],
+                                  stdout=sys.stdout)
+        except subprocess.CalledProcessError:
+            self.result["status"] = Status.ERROR
+            return
 
-    # Choose and check the deployment
+        credentials = deployment.get_credentials_for("openstack")["admin"]
-    call_rally("deployment use --deployment %s" % DEPLOYMENT_NAME)
+        clients = credentials.clients()
-    call_rally("deployment check")
 
-    config = json.loads(
+        if self.args.ctx_create_resources:
-        subprocess.check_output(["rally", "deployment", "config"]))
-    config = config["creds"]["openstack"]
-    config.update(config.pop("admin"))
-    clients = credential.OpenStackCredential(**config).clients()
-
-    if args.ctx_create_resources:
             # If the 'ctx-create-resources' arg is provided, delete images and
             # flavors, and also create a shared network to make Tempest context
             # create needed resources.
@@ -213,89 +205,311 @@ def main():
                     "vcpus": 1,
                     "disk": 0
                 }
-            LOG.info(
+                LOG.info("Creating flavor '%s' with the following properties: "
-                "Creating flavor '%s' with the following properties: RAM "
+                         "RAM = %dMB, VCPUs = 1, disk = 0GB" %
-                "= %dMB, VCPUs = 1, disk = 0GB" % (params["name"], flv_ram))
+                         (params["name"], flv_ram))
                 clients.nova().flavors.create(**params)
 
-    render_vars = dict(verifications=[])
+    def to_html(self):
+        return ""
 
-    # List plugins for verifiers management
-    render_vars["list_plugins"] = call_rally("verify list-plugins")
 
-    # Create a verifier
+class ListPlugins(Step):
-    render_vars["create_verifier"] = call_rally(
+    """List plugins for verifiers management."""
-        "verify create-verifier --type %s --name my-verifier --source %s"
-        % (VERIFIER_TYPE, VERIFIER_SOURCE))
 
-    # Show the verifier
+    COMMAND = "list-plugins"
-    render_vars["show_verifier"] = call_rally("verify show-verifier")
+    DEPENDS_ON = SetUpStep
 
-    # List verifiers
-    render_vars["list_verifiers"] = call_rally("verify list-verifiers")
 
-    # Get verifier ID
+class CreateVerifier(Step):
-    verifier_id = envutils.get_global(envutils.ENV_VERIFIER)
+    """Create a Tempest verifier."""
-    # Get the penultimate verifier commit ID
+
-    repo_dir = os.path.join(
+    COMMAND = ("create-verifier --type %(type)s --name %(name)s "
+               "--source %(source)s")
+    DEPENDS_ON = ListPlugins
+    CALL_ARGS = {"type": "tempest",
+                 "name": "my-verifier",
+                 "source": "https://git.openstack.org/openstack/tempest"}
+
+
+class ShowVerifier(Step):
+    """Show information about the created verifier."""
+
+    COMMAND = "show-verifier"
+    DEPENDS_ON = CreateVerifier
+
+
+class ListVerifiers(Step):
+    """List all installed verifiers."""
+
+    COMMAND = "list-verifiers"
+    DEPENDS_ON = CreateVerifier
+
+
+class UpdateVerifier(Step):
+    """Switch the verifier to the penultimate version."""
+
+    COMMAND = "update-verifier --version %(version)s --update-venv"
+    DEPENDS_ON = CreateVerifier
+
+    def setUp(self):
+        """Obtain penultimate verifier commit for downgrading to it"""
+        verifier_id = self.rapi.verifier.list()[0]["uuid"]
+        verifications_dir = os.path.join(
             os.path.expanduser("~"),
             ".rally/verification/verifier-%s/repo" % verifier_id)
+        # Get the penultimate verifier commit ID
         p_commit_id = subprocess.check_output(
-        ["git", "log", "-n", "1", "--pretty=format:%H"], cwd=repo_dir).strip()
+            ["git", "log", "-n", "1", "--pretty=format:%H"],
-    # Switch the verifier to the penultimate version
+            cwd=verifications_dir).strip()
-    render_vars["update_verifier"] = call_rally(
+        self.CALL_ARGS = {"version": p_commit_id}
-        "verify update-verifier --version %s --update-venv" % p_commit_id)
 
-    # Generate and show the verifier config file
-    render_vars["configure_verifier"] = call_rally(
-        "verify configure-verifier --show")
 
-    # Add a verifier extension
+class ConfigureVerifier(Step):
-    render_vars["add_verifier_ext"] = call_rally(
+    """Generate and show the verifier config file."""
-        "verify add-verifier-ext --source %s" % VERIFIER_EXT_REPO)
 
-    # List verifier extensions
+    COMMAND = "configure-verifier --show"
-    render_vars["list_verifier_exts"] = call_rally("verify list-verifier-exts")
+    DEPENDS_ON = CreateVerifier
 
-    # List verifier tests
-    render_vars["list_verifier_tests"] = call_rally(
-        "verify list-verifier-tests %s" % MODES[args.mode])
 
+class ExtendVerifier(Step):
+    """Extend verifier with keystone integration tests."""
+
+    COMMAND = "add-verifier-ext --source %(source)s"
+    DEPENDS_ON = CreateVerifier
+    CALL_ARGS = {"source": "https://git.openstack.org/openstack/keystone"}
+
+
+class ListVerifierExtensions(Step):
+    """List all extensions of verifier."""
+
+    COMMAND = "list-verifier-exts"
+    DEPENDS_ON = ExtendVerifier
+
+
+class ListVerifierTests(Step):
+    """List all tests of specific verifier."""
+
+    COMMAND = "list-verifier-tests"
+    DEPENDS_ON = CreateVerifier
+
+
+class RunVerification(Step):
+    """Run a verification."""
+
+    DEPENDS_ON = ConfigureVerifier
+    COMMAND = ("start --pattern set=%(set)s --skip-list %(skip_tests)s "
+               "--xfail-list %(xfail_tests)s --tag %(tag)s %(set)s-set "
+               "--detailed")
+    SKIP_TESTS = {
+        "tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON."
+        "test_get_flavor[id-1f12046b-753d-40d2-abb6-d8eb8b30cb2f,smoke]":
+            "This test was skipped intentionally"}
+    XFAIL_TESTS = {
+        "tempest.api.compute.servers.test_server_actions."
+        "ServerActionsTestJSON.test_get_vnc_console"
+        "[id-c6bc11bf-592e-4015-9319-1c98dc64daf5]":
+            "This test fails because 'novnc' console type is unavailable"}
+
+    def setUp(self):
+        self.CALL_ARGS["tag"] = "tag-1 tag-2"
+        self.CALL_ARGS["set"] = "full" if self.args.mode == "full" else "smoke"
         # Start a verification, show results and generate reports
-    skip_list_path = write_file("skip-list.yaml", SKIP_TESTS)
+        skip_tests = json.dumps(self.SKIP_TESTS)
-    xfail_list_path = write_file("xfail-list.yaml", XFAIL_TESTS)
+        xfail_tests = json.dumps(self.XFAIL_TESTS)
-    run_args = ("%s --skip-list %s --xfail-list %s --tag first-run %s-set "
+        self.CALL_ARGS["skip_tests"] = self._write_file(
-                "--detailed" % (MODES[args.mode], skip_list_path,
+            self._generate_path("skip-list.json"), skip_tests)
-                                xfail_list_path, args.mode))
+        self.CALL_ARGS["xfail_tests"] = self._write_file(
-    render_vars["verifications"].append(start_verification(run_args))
+            self._generate_path("xfail-list.json"), xfail_tests)
+
+    def run(self):
+        super(RunVerification, self).run()
+        if "Success: 0" in self.result["output"]:
+            self.result["status"] = Status.FAILURE
+
+
+class ReRunVerification(RunVerification):
+    """Re-Run previous verification."""
+
+    COMMAND = "rerun --tag one-more-attempt"
+
+    def run(self):
+        super(RunVerification, self).run()
+        if "Success: 0" in self.result["output"]:
+            self.result["status"] = Status.FAILURE
+
+
+class ShowVerification(Step):
+    """Show results of verification."""
+
+    COMMAND = "show"
+    DEPENDS_ON = RunVerification
+
+
+class ShowSecondVerification(ShowVerification):
+    """Show results of verification."""
+
+    DEPENDS_ON = ReRunVerification
+
+
+class ShowDetailedVerification(Step):
+    """Show detailed results of verification."""
+
+    COMMAND = "show --detailed"
+    DEPENDS_ON = RunVerification
+
+
+class ShowDetailedSecondVerification(ShowDetailedVerification):
+    """Show detailed results of verification."""
+
+    DEPENDS_ON = ReRunVerification
+
+
+class ReportVerificationMixin(Step):
+    """Mixin for obtaining reports of verifications."""
+
+    COMMAND = "report --uuid %(uuids)s --type %(type)s --to %(out)s"
+
+    HTML_TEMPLATE = ("<span class=\"%(status)s\">[%(status)s]</span>\n"
+                     "<a href=\"%(out)s\">%(doc)s</a> "
+                     "[<a href=\"%(output_file)s\">Output from CLI</a>]\n"
+                     "<code>$ %(cmd)s</code>")
+
+    def setUp(self):
+        self.CALL_ARGS["out"] = "<path>"
+        self.CALL_ARGS["uuids"] = "<uuid-1> <uuid-2>"
+        cmd = self.COMMAND % self.CALL_ARGS
+        report = "%s.%s" % (cmd.replace("/", "_").replace(" ", "_"),
+                            self.CALL_ARGS["type"])
+        print(report)
+        self.CALL_ARGS["out"] = self._generate_path(report)
+        self.CALL_ARGS["uuids"] = " ".join(
+            [v["uuid"] for v in self.rapi.verification.list()])
+        print(self.COMMAND % self.CALL_ARGS)
+
+    def run(self):
+        super(ReportVerificationMixin, self).run()
+        creport = "%s.gz" % self.CALL_ARGS["out"]
+        with open(self.CALL_ARGS["out"], "rb") as f_in:
+            with gzip.open(creport, "wb") as f_out:
+                f_out.writelines(f_in)
+        self.result["out"] = creport
+
+
+class HtmlVerificationReport(ReportVerificationMixin):
+    """Generate HTML report for verification(s)."""
+
+    CALL_ARGS = {"type": "html"}
+    DEPENDS_ON = RunVerification
+
+
+class JsonVerificationReport(ReportVerificationMixin):
+    """Generate JSON report for verification(s)."""
+
+    CALL_ARGS = {"type": "json"}
+    DEPENDS_ON = RunVerification
+
+
+class JunitVerificationReport(ReportVerificationMixin):
+    """Generate JUNIT report for verification(s)."""
+
+    CALL_ARGS = {"type": "junit-xml"}
+    DEPENDS_ON = RunVerification
+
+
+class ListVerifications(Step):
+    """List all verifications."""
+
+    COMMAND = "list"
+    DEPENDS_ON = CreateVerifier
+
+
+class DeleteVerifierExtension(Step):
+    """Delete keystone extension."""
+
+    COMMAND = "delete-verifier-ext --name %(name)s"
+    CALL_ARGS = {"name": "keystone_tests"}
+    DEPENDS_ON = ExtendVerifier
+
+
+class DeleteVerifier(Step):
+    """Delete Tempest verifier and all verifications."""
+
+    COMMAND = "delete-verifier --id %(id)s --force"
+    CALL_ARGS = {"id": CreateVerifier.CALL_ARGS["name"]}
+    DEPENDS_ON = CreateVerifier
+
+
+def run(args):
+
+    steps = [SetUpStep,
+             ListPlugins,
+             CreateVerifier,
+             ShowVerifier,
+             ListVerifiers,
+             UpdateVerifier,
+             ConfigureVerifier,
+             ExtendVerifier,
+             ListVerifierExtensions,
+             ListVerifierTests,
+             RunVerification,
+             ShowVerification,
+             ShowDetailedVerification,
+             HtmlVerificationReport,
+             JsonVerificationReport,
+             JunitVerificationReport,
+             ListVerifications,
+             DeleteVerifierExtension,
+             DeleteVerifier]
 
     if args.compare:
-        # Start another verification, show results and generate reports
+        # need to launch one more verification
-        with gzip.open(render_vars["list_verifier_tests"]["stdout_file"]) as f:
+        place_to_insert = steps.index(ShowDetailedVerification) + 1
-            tests = [t for t in f.read().split("\n") if TEST_NAME_RE.match(t)]
+        # insert steps in reverse order to be able to use the same index
-            load_list_path = write_file("load-list.txt", "\n".join(tests))
+        steps.insert(place_to_insert, ShowDetailedSecondVerification)
-        run_args = "--load-list %s --tag second-run %s-set --detailed" % (
+        steps.insert(place_to_insert, ShowSecondVerification)
-            load_list_path, args.mode)
+        steps.insert(place_to_insert, ReRunVerification)
-        render_vars["verifications"].append(start_verification(run_args))
 
-        # Generate trends reports for two verifications
+    results = collections.OrderedDict()
-        render_vars["compare"] = generate_trends_reports(
+    rapi = api.API()
-            render_vars["verifications"][-2]["uuid"],
+    for step_cls in steps:
-            render_vars["verifications"][-1]["uuid"])
+        step = step_cls(args, rapi=rapi)
+        if step.check(results):
+            step.run()
+        results[step_cls] = step
 
-    # List verifications
+    return results.values()
-    render_vars["list"] = call_rally("verify list")
 
-    # Delete the verifier extension
-    render_vars["delete_verifier_ext"] = call_rally(
-        "verify delete-verifier-ext --name %s" % VERIFIER_EXT_NAME)
-    # Delete the verifier and all verifications
-    render_vars["delete_verifier"] = call_rally(
-        "verify delete-verifier --id %s --force" % verifier_id)
 
-    render_page(**render_vars)
+def main():
+    parser = argparse.ArgumentParser(description="Launch rally-verify job.")
+    parser.add_argument("--mode", type=str, default="light",
+                        help="Mode of job. The 'full' mode corresponds to the "
+                             "full set of verifier tests. The 'light' mode "
+                             "corresponds to the smoke set of verifier tests.",
+                        choices=["light", "full"])
+    parser.add_argument("--compare", action="store_true",
+                        help="Start the second verification to generate a "
+                             "trends report for two verifications.")
+    # TODO(ylobankov): Remove hard-coded Tempest related things and make it
+    #                  configurable.
+    parser.add_argument("--ctx-create-resources", action="store_true",
+                        help="Make Tempest context create needed resources "
+                             "for the tests.")
 
-    return _return_status
+    args = parser.parse_args()
+
+    steps = run(args)
+    results = [step.to_html() for step in steps]
+
+    template = utils.get_template("ci/index_verify.html")
+    with open(os.path.join(Step.BASE_DIR, "extra/index.html"), "w") as f:
+        f.write(template.render(steps=results))
+
+    if len([None for step in steps if step.result["status"] in (
+            Status.PASS, Status.FAILURE)]) == len(steps):
+        return 0
+    return 1
 
 if __name__ == "__main__":
     sys.exit(main())