From f0fa427d35d8de6d0436bce0bb023c1a81699630 Mon Sep 17 00:00:00 2001 From: Andrey Kurilin Date: Fri, 9 Jun 2017 20:47:02 +0300 Subject: [PATCH] make verification job great again Change-Id: I56f068bd803f1a11ddf6534b35af34c168037be9 --- tests/ci/rally_verify.py | 684 +++++++++++++++++++++++++-------------- 1 file changed, 449 insertions(+), 235 deletions(-) diff --git a/tests/ci/rally_verify.py b/tests/ci/rally_verify.py index e85cf05e..74131189 100755 --- a/tests/ci/rally_verify.py +++ b/tests/ci/rally_verify.py @@ -13,6 +13,7 @@ # under the License. import argparse +import collections import gzip import json import logging @@ -22,126 +23,462 @@ import subprocess import sys import uuid -from rally.cli import envutils -from rally.plugins.openstack import credential +from rally import api from rally.ui import utils LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) -BASE_DIR = "rally-verify" - -MODES = {"full": "--pattern set=full", "light": "--pattern set=smoke"} -DEPLOYMENT_NAME = "devstack" -VERIFIER_TYPE = "tempest" -VERIFIER_SOURCE = "https://git.openstack.org/openstack/tempest" -VERIFIER_EXT_REPO = "https://git.openstack.org/openstack/keystone" -VERIFIER_EXT_NAME = "keystone_tests" -SKIP_TESTS = ( - "tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON." - "test_get_flavor[id-1f12046b-753d-40d2-abb6-d8eb8b30cb2f,smoke]: " - "This test was skipped intentionally") -XFAIL_TESTS = ( - "tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON." - "test_get_vnc_console[id-c6bc11bf-592e-4015-9319-1c98dc64daf5]: " - "This test fails because 'novnc' console type is unavailable") -TEST_NAME_RE = re.compile(r"^[a-zA-Z_.0-9]+(\[[a-zA-Z-,=0-9]*\])?$") - # NOTE(andreykurilin): this variable is used to generate output file names # with prefix ${CALL_COUNT}_ . _call_count = 0 -# NOTE(andreykurilin): if some command fails, script should end with -# error status -_return_status = 0 -def call_rally(cmd, print_output=False, output_type=None): - """Execute a Rally command and write result in files.""" - global _return_status - global _call_count - _call_count += 1 - - data = {"cmd": "rally --rally-debug %s" % cmd} - stdout_file = "{base_dir}/{prefix}_{cmd}.txt.gz" - - cmd = cmd.replace("/", "_") - data.update({"stdout_file": stdout_file.format(base_dir=BASE_DIR, - prefix=_call_count, - cmd=cmd.replace(" ", "_"))}) - - if output_type: - data["output_file"] = data["stdout_file"].replace( - ".txt.", ".%s." % output_type) - data["cmd"] += " --to %s" % data["output_file"] - data["cmd"] += " --type %s" % output_type - - try: - LOG.info("Try to execute `%s`." % data["cmd"]) - stdout = subprocess.check_output(data["cmd"].split(), - stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - LOG.error("Command `%s` failed." % data["cmd"]) - stdout = e.output - data["status"] = "fail" - _return_status = 1 - else: - data["status"] = "success" - - if output_type: - # let's gzip results - with open(data["output_file"]) as f: - output = f.read() - with gzip.open(data["output_file"], "wb") as f: - f.write(output) - - stdout = "$ %s\n%s" % (data["cmd"], stdout) - - with gzip.open(data["stdout_file"], "wb") as f: - f.write(stdout) - - if print_output: - print(stdout) - - return data +class Status(object): + PASS = "success" + ERROR = "error" + SKIPPED = "skip" + FAILURE = "fail" -def start_verification(args): - """Start a verification, show results and generate reports.""" - results = call_rally("verify start %s" % args) - results["uuid"] = envutils.get_global(envutils.ENV_VERIFICATION) - results["show"] = call_rally("verify show") - results["show_detailed"] = call_rally("verify show --detailed") - for output_type in ("json", "html", "junit-xml"): - results[output_type.replace("-", "_")] = call_rally( - "verify report", output_type=output_type) - # NOTE(andreykurilin): we need to clean verification uuid from global - # environment to be able to load it next time(for another verification). - envutils.clear_global(envutils.ENV_VERIFICATION) - return results +class Step(object): + COMMAND = None + DEPENDS_ON = None + CALL_ARGS = {} + + BASE_DIR = "rally-verify" + HTML_TEMPLATE = ("[%(status)s]\n" + "%(doc)s\n" + "$ %(cmd)s") + + def __init__(self, args, rapi): + self.args = args + self.rapi = rapi + self.result = {"status": Status.PASS, + "doc": self.__doc__} + + @property + def name(self): + return " ".join(re.findall("[A-Z][^A-Z]*", + self.__class__.__name__)).lower() + + def check(self, results): + """Check weather this step should be executed or skipped.""" + if self.DEPENDS_ON is not None: + if results[self.DEPENDS_ON].result["status"] in ( + Status.PASS, Status.FAILURE): + return True + else: + self.result["status"] = Status.SKIPPED + msg = ("Step '%s' is skipped, since depends on step '%s' is " + "skipped or finished with an error." % + (self.name, results[self.DEPENDS_ON].name)) + stdout_file = self._generate_path( + "%s.txt.gz" % self.__class__.__name__) + + self.result["output_file"] = self._write_file( + stdout_file, msg, compress=True) + return False + return True + + def setUp(self): + """Obtain variables required for execution""" + pass + + def run(self): + """Execute step. The default action - execute the command""" + self.setUp() + + cmd = "rally --rally-debug verify %s" % (self.COMMAND % self.CALL_ARGS) + self.result["cmd"] = cmd + self.result["status"], self.result["output"] = self.call_rally(cmd) + + stdout_file = self._generate_path("%s.txt.gz" % cmd) + self.result["output_file"] = self._write_file( + stdout_file, self.result["output"], compress=True) + + @classmethod + def _generate_path(cls, root): + global _call_count + _call_count += 1 + + root = root.replace("<", "").replace(">", "").replace("/", "_") + parts = ["%s" % _call_count] + for path in root.split(" "): + if path.startswith(cls.BASE_DIR): + path = path[len(cls.BASE_DIR) + 1:] + parts.append(path) + return os.path.join(cls.BASE_DIR, "_".join(parts)) + + @classmethod + def _write_file(cls, path, data, compress=False): + """Create a file and write some data to it.""" + if compress: + with gzip.open(path, "wb") as f: + f.write(data) + else: + with open(path, "wb") as f: + f.write(data) + return path + + @staticmethod + def call_rally(command): + """Execute a Rally verify command.""" + try: + LOG.info("Try to execute `%s`." % command) + stdout = subprocess.check_output(command.split(), + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + LOG.error("Command `%s` failed." % command) + return Status.ERROR, e.output + else: + return Status.PASS, stdout + + def to_html(self): + return self.HTML_TEMPLATE % self.result -def write_file(filename, data): - """Create a file and write some data to it.""" - path = os.path.join(BASE_DIR, filename) - with open(path, "wb") as f: - f.write(data) - return path +class SetUpStep(Step): + """Validate deployment, create required resources and directories.""" + + DEPLOYMENT_NAME = "devstack" + + def run(self): + if not os.path.exists("%s/extra" % self.BASE_DIR): + os.makedirs("%s/extra" % self.BASE_DIR) + + # ensure that deployment exit + deployment = self.rapi.deployment._get(self.DEPLOYMENT_NAME) + # check it + result = self.rapi.deployment.check( + deployment=self.DEPLOYMENT_NAME)["openstack"] + if "admin_error" in result[0] or "user_error" in result[0]: + self.result["status"] = Status.ERROR + return + + try: + subprocess.check_call(["rally", "deployment", "use", + "--deployment", self.DEPLOYMENT_NAME], + stdout=sys.stdout) + except subprocess.CalledProcessError: + self.result["status"] = Status.ERROR + return + + credentials = deployment.get_credentials_for("openstack")["admin"] + clients = credentials.clients() + + if self.args.ctx_create_resources: + # If the 'ctx-create-resources' arg is provided, delete images and + # flavors, and also create a shared network to make Tempest context + # create needed resources. + LOG.info("The 'ctx-create-resources' arg is provided. Deleting " + "images and flavors, and also creating a shared network " + "to make Tempest context create needed resources.") + + LOG.info("Deleting images.") + for image in clients.glance().images.list(): + clients.glance().images.delete(image.id) + + LOG.info("Deleting flavors.") + for flavor in clients.nova().flavors.list(): + clients.nova().flavors.delete(flavor.id) + + LOG.info("Creating a shared network.") + net_body = { + "network": { + "name": "shared-net-%s" % str(uuid.uuid4()), + "tenant_id": clients.keystone.auth_ref.project_id, + "shared": True + } + } + clients.neutron().create_network(net_body) + else: + # Otherwise, just in case create only flavors with the following + # properties: RAM = 64MB and 128MB, VCPUs = 1, disk = 0GB to make + # Tempest context discover them. + LOG.info("The 'ctx-create-resources' arg is not provided. " + "Creating flavors to make Tempest context discover them.") + for flv_ram in [64, 128]: + params = { + "name": "flavor-%s" % str(uuid.uuid4()), + "ram": flv_ram, + "vcpus": 1, + "disk": 0 + } + LOG.info("Creating flavor '%s' with the following properties: " + "RAM = %dMB, VCPUs = 1, disk = 0GB" % + (params["name"], flv_ram)) + clients.nova().flavors.create(**params) + + def to_html(self): + return "" -def generate_trends_reports(uuid_1, uuid_2): - """Generate trends reports.""" - results = {} - for output_type in ("json", "html", "junit-xml"): - results[output_type.replace("-", "_")] = call_rally( - "verify report --uuid %s %s" % (uuid_1, uuid_2), - output_type=output_type) - return results +class ListPlugins(Step): + """List plugins for verifiers management.""" + + COMMAND = "list-plugins" + DEPENDS_ON = SetUpStep -def render_page(**render_vars): - template = utils.get_template("ci/index_verify.html") - with open(os.path.join(BASE_DIR, "extra/index.html"), "w") as f: - f.write(template.render(**render_vars)) +class CreateVerifier(Step): + """Create a Tempest verifier.""" + + COMMAND = ("create-verifier --type %(type)s --name %(name)s " + "--source %(source)s") + DEPENDS_ON = ListPlugins + CALL_ARGS = {"type": "tempest", + "name": "my-verifier", + "source": "https://git.openstack.org/openstack/tempest"} + + +class ShowVerifier(Step): + """Show information about the created verifier.""" + + COMMAND = "show-verifier" + DEPENDS_ON = CreateVerifier + + +class ListVerifiers(Step): + """List all installed verifiers.""" + + COMMAND = "list-verifiers" + DEPENDS_ON = CreateVerifier + + +class UpdateVerifier(Step): + """Switch the verifier to the penultimate version.""" + + COMMAND = "update-verifier --version %(version)s --update-venv" + DEPENDS_ON = CreateVerifier + + def setUp(self): + """Obtain penultimate verifier commit for downgrading to it""" + verifier_id = self.rapi.verifier.list()[0]["uuid"] + verifications_dir = os.path.join( + os.path.expanduser("~"), + ".rally/verification/verifier-%s/repo" % verifier_id) + # Get the penultimate verifier commit ID + p_commit_id = subprocess.check_output( + ["git", "log", "-n", "1", "--pretty=format:%H"], + cwd=verifications_dir).strip() + self.CALL_ARGS = {"version": p_commit_id} + + +class ConfigureVerifier(Step): + """Generate and show the verifier config file.""" + + COMMAND = "configure-verifier --show" + DEPENDS_ON = CreateVerifier + + +class ExtendVerifier(Step): + """Extend verifier with keystone integration tests.""" + + COMMAND = "add-verifier-ext --source %(source)s" + DEPENDS_ON = CreateVerifier + CALL_ARGS = {"source": "https://git.openstack.org/openstack/keystone"} + + +class ListVerifierExtensions(Step): + """List all extensions of verifier.""" + + COMMAND = "list-verifier-exts" + DEPENDS_ON = ExtendVerifier + + +class ListVerifierTests(Step): + """List all tests of specific verifier.""" + + COMMAND = "list-verifier-tests" + DEPENDS_ON = CreateVerifier + + +class RunVerification(Step): + """Run a verification.""" + + DEPENDS_ON = ConfigureVerifier + COMMAND = ("start --pattern set=%(set)s --skip-list %(skip_tests)s " + "--xfail-list %(xfail_tests)s --tag %(tag)s %(set)s-set " + "--detailed") + SKIP_TESTS = { + "tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON." + "test_get_flavor[id-1f12046b-753d-40d2-abb6-d8eb8b30cb2f,smoke]": + "This test was skipped intentionally"} + XFAIL_TESTS = { + "tempest.api.compute.servers.test_server_actions." + "ServerActionsTestJSON.test_get_vnc_console" + "[id-c6bc11bf-592e-4015-9319-1c98dc64daf5]": + "This test fails because 'novnc' console type is unavailable"} + + def setUp(self): + self.CALL_ARGS["tag"] = "tag-1 tag-2" + self.CALL_ARGS["set"] = "full" if self.args.mode == "full" else "smoke" + # Start a verification, show results and generate reports + skip_tests = json.dumps(self.SKIP_TESTS) + xfail_tests = json.dumps(self.XFAIL_TESTS) + self.CALL_ARGS["skip_tests"] = self._write_file( + self._generate_path("skip-list.json"), skip_tests) + self.CALL_ARGS["xfail_tests"] = self._write_file( + self._generate_path("xfail-list.json"), xfail_tests) + + def run(self): + super(RunVerification, self).run() + if "Success: 0" in self.result["output"]: + self.result["status"] = Status.FAILURE + + +class ReRunVerification(RunVerification): + """Re-Run previous verification.""" + + COMMAND = "rerun --tag one-more-attempt" + + def run(self): + super(RunVerification, self).run() + if "Success: 0" in self.result["output"]: + self.result["status"] = Status.FAILURE + + +class ShowVerification(Step): + """Show results of verification.""" + + COMMAND = "show" + DEPENDS_ON = RunVerification + + +class ShowSecondVerification(ShowVerification): + """Show results of verification.""" + + DEPENDS_ON = ReRunVerification + + +class ShowDetailedVerification(Step): + """Show detailed results of verification.""" + + COMMAND = "show --detailed" + DEPENDS_ON = RunVerification + + +class ShowDetailedSecondVerification(ShowDetailedVerification): + """Show detailed results of verification.""" + + DEPENDS_ON = ReRunVerification + + +class ReportVerificationMixin(Step): + """Mixin for obtaining reports of verifications.""" + + COMMAND = "report --uuid %(uuids)s --type %(type)s --to %(out)s" + + HTML_TEMPLATE = ("[%(status)s]\n" + "%(doc)s " + "[Output from CLI]\n" + "$ %(cmd)s") + + def setUp(self): + self.CALL_ARGS["out"] = "" + self.CALL_ARGS["uuids"] = " " + cmd = self.COMMAND % self.CALL_ARGS + report = "%s.%s" % (cmd.replace("/", "_").replace(" ", "_"), + self.CALL_ARGS["type"]) + print(report) + self.CALL_ARGS["out"] = self._generate_path(report) + self.CALL_ARGS["uuids"] = " ".join( + [v["uuid"] for v in self.rapi.verification.list()]) + print(self.COMMAND % self.CALL_ARGS) + + def run(self): + super(ReportVerificationMixin, self).run() + creport = "%s.gz" % self.CALL_ARGS["out"] + with open(self.CALL_ARGS["out"], "rb") as f_in: + with gzip.open(creport, "wb") as f_out: + f_out.writelines(f_in) + self.result["out"] = creport + + +class HtmlVerificationReport(ReportVerificationMixin): + """Generate HTML report for verification(s).""" + + CALL_ARGS = {"type": "html"} + DEPENDS_ON = RunVerification + + +class JsonVerificationReport(ReportVerificationMixin): + """Generate JSON report for verification(s).""" + + CALL_ARGS = {"type": "json"} + DEPENDS_ON = RunVerification + + +class JunitVerificationReport(ReportVerificationMixin): + """Generate JUNIT report for verification(s).""" + + CALL_ARGS = {"type": "junit-xml"} + DEPENDS_ON = RunVerification + + +class ListVerifications(Step): + """List all verifications.""" + + COMMAND = "list" + DEPENDS_ON = CreateVerifier + + +class DeleteVerifierExtension(Step): + """Delete keystone extension.""" + + COMMAND = "delete-verifier-ext --name %(name)s" + CALL_ARGS = {"name": "keystone_tests"} + DEPENDS_ON = ExtendVerifier + + +class DeleteVerifier(Step): + """Delete Tempest verifier and all verifications.""" + + COMMAND = "delete-verifier --id %(id)s --force" + CALL_ARGS = {"id": CreateVerifier.CALL_ARGS["name"]} + DEPENDS_ON = CreateVerifier + + +def run(args): + + steps = [SetUpStep, + ListPlugins, + CreateVerifier, + ShowVerifier, + ListVerifiers, + UpdateVerifier, + ConfigureVerifier, + ExtendVerifier, + ListVerifierExtensions, + ListVerifierTests, + RunVerification, + ShowVerification, + ShowDetailedVerification, + HtmlVerificationReport, + JsonVerificationReport, + JunitVerificationReport, + ListVerifications, + DeleteVerifierExtension, + DeleteVerifier] + + if args.compare: + # need to launch one more verification + place_to_insert = steps.index(ShowDetailedVerification) + 1 + # insert steps in reverse order to be able to use the same index + steps.insert(place_to_insert, ShowDetailedSecondVerification) + steps.insert(place_to_insert, ShowSecondVerification) + steps.insert(place_to_insert, ReRunVerification) + + results = collections.OrderedDict() + rapi = api.API() + for step_cls in steps: + step = step_cls(args, rapi=rapi) + if step.check(results): + step.run() + results[step_cls] = step + + return results.values() def main(): @@ -150,7 +487,7 @@ def main(): help="Mode of job. The 'full' mode corresponds to the " "full set of verifier tests. The 'light' mode " "corresponds to the smoke set of verifier tests.", - choices=MODES.keys()) + choices=["light", "full"]) parser.add_argument("--compare", action="store_true", help="Start the second verification to generate a " "trends report for two verifications.") @@ -162,140 +499,17 @@ def main(): args = parser.parse_args() - if not os.path.exists("%s/extra" % BASE_DIR): - os.makedirs("%s/extra" % BASE_DIR) + steps = run(args) + results = [step.to_html() for step in steps] - # Choose and check the deployment - call_rally("deployment use --deployment %s" % DEPLOYMENT_NAME) - call_rally("deployment check") + template = utils.get_template("ci/index_verify.html") + with open(os.path.join(Step.BASE_DIR, "extra/index.html"), "w") as f: + f.write(template.render(steps=results)) - config = json.loads( - subprocess.check_output(["rally", "deployment", "config"])) - config = config["creds"]["openstack"] - config.update(config.pop("admin")) - clients = credential.OpenStackCredential(**config).clients() - - if args.ctx_create_resources: - # If the 'ctx-create-resources' arg is provided, delete images and - # flavors, and also create a shared network to make Tempest context - # create needed resources. - LOG.info("The 'ctx-create-resources' arg is provided. Deleting " - "images and flavors, and also creating a shared network " - "to make Tempest context create needed resources.") - - LOG.info("Deleting images.") - for image in clients.glance().images.list(): - clients.glance().images.delete(image.id) - - LOG.info("Deleting flavors.") - for flavor in clients.nova().flavors.list(): - clients.nova().flavors.delete(flavor.id) - - LOG.info("Creating a shared network.") - net_body = { - "network": { - "name": "shared-net-%s" % str(uuid.uuid4()), - "tenant_id": clients.keystone.auth_ref.project_id, - "shared": True - } - } - clients.neutron().create_network(net_body) - else: - # Otherwise, just in case create only flavors with the following - # properties: RAM = 64MB and 128MB, VCPUs = 1, disk = 0GB to make - # Tempest context discover them. - LOG.info("The 'ctx-create-resources' arg is not provided. " - "Creating flavors to make Tempest context discover them.") - for flv_ram in [64, 128]: - params = { - "name": "flavor-%s" % str(uuid.uuid4()), - "ram": flv_ram, - "vcpus": 1, - "disk": 0 - } - LOG.info( - "Creating flavor '%s' with the following properties: RAM " - "= %dMB, VCPUs = 1, disk = 0GB" % (params["name"], flv_ram)) - clients.nova().flavors.create(**params) - - render_vars = dict(verifications=[]) - - # List plugins for verifiers management - render_vars["list_plugins"] = call_rally("verify list-plugins") - - # Create a verifier - render_vars["create_verifier"] = call_rally( - "verify create-verifier --type %s --name my-verifier --source %s" - % (VERIFIER_TYPE, VERIFIER_SOURCE)) - - # Show the verifier - render_vars["show_verifier"] = call_rally("verify show-verifier") - - # List verifiers - render_vars["list_verifiers"] = call_rally("verify list-verifiers") - - # Get verifier ID - verifier_id = envutils.get_global(envutils.ENV_VERIFIER) - # Get the penultimate verifier commit ID - repo_dir = os.path.join( - os.path.expanduser("~"), - ".rally/verification/verifier-%s/repo" % verifier_id) - p_commit_id = subprocess.check_output( - ["git", "log", "-n", "1", "--pretty=format:%H"], cwd=repo_dir).strip() - # Switch the verifier to the penultimate version - render_vars["update_verifier"] = call_rally( - "verify update-verifier --version %s --update-venv" % p_commit_id) - - # Generate and show the verifier config file - render_vars["configure_verifier"] = call_rally( - "verify configure-verifier --show") - - # Add a verifier extension - render_vars["add_verifier_ext"] = call_rally( - "verify add-verifier-ext --source %s" % VERIFIER_EXT_REPO) - - # List verifier extensions - render_vars["list_verifier_exts"] = call_rally("verify list-verifier-exts") - - # List verifier tests - render_vars["list_verifier_tests"] = call_rally( - "verify list-verifier-tests %s" % MODES[args.mode]) - - # Start a verification, show results and generate reports - skip_list_path = write_file("skip-list.yaml", SKIP_TESTS) - xfail_list_path = write_file("xfail-list.yaml", XFAIL_TESTS) - run_args = ("%s --skip-list %s --xfail-list %s --tag first-run %s-set " - "--detailed" % (MODES[args.mode], skip_list_path, - xfail_list_path, args.mode)) - render_vars["verifications"].append(start_verification(run_args)) - - if args.compare: - # Start another verification, show results and generate reports - with gzip.open(render_vars["list_verifier_tests"]["stdout_file"]) as f: - tests = [t for t in f.read().split("\n") if TEST_NAME_RE.match(t)] - load_list_path = write_file("load-list.txt", "\n".join(tests)) - run_args = "--load-list %s --tag second-run %s-set --detailed" % ( - load_list_path, args.mode) - render_vars["verifications"].append(start_verification(run_args)) - - # Generate trends reports for two verifications - render_vars["compare"] = generate_trends_reports( - render_vars["verifications"][-2]["uuid"], - render_vars["verifications"][-1]["uuid"]) - - # List verifications - render_vars["list"] = call_rally("verify list") - - # Delete the verifier extension - render_vars["delete_verifier_ext"] = call_rally( - "verify delete-verifier-ext --name %s" % VERIFIER_EXT_NAME) - # Delete the verifier and all verifications - render_vars["delete_verifier"] = call_rally( - "verify delete-verifier --id %s --force" % verifier_id) - - render_page(**render_vars) - - return _return_status + if len([None for step in steps if step.result["status"] in ( + Status.PASS, Status.FAILURE)]) == len(steps): + return 0 + return 1 if __name__ == "__main__": sys.exit(main())