203 lines
6.5 KiB
Python
Executable File
203 lines
6.5 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import argparse
|
|
import gzip
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
import yaml
|
|
|
|
from rally.cli import envutils
|
|
from rally.ui import utils
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
LOG.setLevel(logging.DEBUG)
|
|
|
|
|
|
MODES_PARAMETERS = {
|
|
"full": "--set full",
|
|
"light": "--set smoke"
|
|
}
|
|
|
|
BASE_DIR = "rally-verify"
|
|
|
|
EXPECTED_FAILURES_FILE = "expected_failures.yaml"
|
|
EXPECTED_FAILURES = {
|
|
"tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON."
|
|
"test_get_vnc_console[id-c6bc11bf-592e-4015-9319-1c98dc64daf5]":
|
|
"This test fails because 'novnc' console type is unavailable."
|
|
}
|
|
|
|
# NOTE(andreykurilin): this variable is used to generate output file names
|
|
# with prefix ${CALL_COUNT}_ .
|
|
_call_count = 0
|
|
# NOTE(andreykurilin): if some command fails, script should end with
|
|
# error status
|
|
_return_status = 0
|
|
|
|
|
|
def call_rally(cmd, print_output=False, output_type=None):
|
|
global _return_status
|
|
global _call_count
|
|
_call_count += 1
|
|
|
|
data = {"cmd": "rally --rally-debug %s" % cmd}
|
|
stdout_file = "{base}/{prefix}_{cmd}.txt.gz"
|
|
|
|
if "--xfails-file" in cmd:
|
|
cmd_items = cmd.split()
|
|
for num, item in enumerate(cmd_items):
|
|
if EXPECTED_FAILURES_FILE in item:
|
|
cmd_items[num] = os.path.basename(item)
|
|
break
|
|
cmd = " ".join(cmd_items)
|
|
|
|
data.update({"stdout_file": stdout_file.format(base=BASE_DIR,
|
|
prefix=_call_count,
|
|
cmd=cmd.replace(" ", "_"))})
|
|
|
|
if output_type:
|
|
data["output_file"] = data["stdout_file"].replace(
|
|
".txt.", ".%s." % output_type)
|
|
data["cmd"] += " --%(type)s --output-file %(file)s" % {
|
|
"type": output_type, "file": data["output_file"]}
|
|
|
|
try:
|
|
LOG.info("Try to launch `%s`." % data["cmd"])
|
|
stdout = subprocess.check_output(data["cmd"], shell=True,
|
|
stderr=subprocess.STDOUT)
|
|
except subprocess.CalledProcessError as e:
|
|
LOG.error("Command `%s` is failed." % data["cmd"])
|
|
stdout = e.output
|
|
data["status"] = "fail"
|
|
_return_status = 1
|
|
else:
|
|
data["status"] = "pass"
|
|
|
|
if output_type:
|
|
# lets gzip results
|
|
with open(data["output_file"]) as f:
|
|
output = f.read()
|
|
with gzip.open(data["output_file"], "wb") as f:
|
|
f.write(output)
|
|
|
|
stdout = "$ %s\n%s" % (data["cmd"], stdout)
|
|
|
|
with gzip.open(data["stdout_file"], "wb") as f:
|
|
f.write(stdout)
|
|
|
|
if print_output:
|
|
print(stdout)
|
|
|
|
return data
|
|
|
|
|
|
def create_file_with_xfails():
|
|
"""Create a YAML file with a list of tests that are expected to fail."""
|
|
with open(os.path.join(BASE_DIR, EXPECTED_FAILURES_FILE), "wb") as f:
|
|
yaml.dump(EXPECTED_FAILURES, f, default_flow_style=False)
|
|
|
|
return os.path.join(os.getcwd(), BASE_DIR, EXPECTED_FAILURES_FILE)
|
|
|
|
|
|
def launch_verification_once(launch_parameters):
|
|
"""Launch verification and show results in different formats."""
|
|
results = call_rally("verify start %s" % launch_parameters)
|
|
results["uuid"] = envutils.get_global(envutils.ENV_VERIFICATION)
|
|
results["result_in_html"] = call_rally("verify results",
|
|
output_type="html")
|
|
results["result_in_json"] = call_rally("verify results",
|
|
output_type="json")
|
|
results["show"] = call_rally("verify show")
|
|
results["show_detailed"] = call_rally("verify show --detailed")
|
|
# NOTE(andreykurilin): we need to clean verification uuid from global
|
|
# environment to be able to load it next time(for another verification).
|
|
envutils.clear_global(envutils.ENV_VERIFICATION)
|
|
return results
|
|
|
|
|
|
def do_compare(uuid_1, uuid_2):
|
|
"""Compare and save results in different formats."""
|
|
results = {}
|
|
for output_format in ("csv", "html", "json"):
|
|
cmd = "verify compare --uuid-1 %(uuid-1)s --uuid-2 %(uuid-2)s" % {
|
|
"uuid-1": uuid_1,
|
|
"uuid-2": uuid_2
|
|
}
|
|
results[output_format] = call_rally(cmd, output_type=output_format)
|
|
return results
|
|
|
|
|
|
def render_page(**render_vars):
|
|
template = utils.get_template("ci/index_verify.html")
|
|
with open(os.path.join(BASE_DIR, "extra/index.html"), "w") as f:
|
|
f.write(template.render(**render_vars))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Launch rally-verify job.")
|
|
parser.add_argument(
|
|
"--mode",
|
|
type=str,
|
|
default="light",
|
|
help="mode of job",
|
|
choices=MODES_PARAMETERS.keys())
|
|
parser.add_argument(
|
|
"--compare",
|
|
action="store_true",
|
|
help="Launch 2 verification and compare them.")
|
|
args = parser.parse_args()
|
|
|
|
if not os.path.exists("%s/extra" % BASE_DIR):
|
|
os.makedirs("%s/extra" % BASE_DIR)
|
|
|
|
# Check deployment
|
|
call_rally("deployment use --deployment devstack", print_output=True)
|
|
call_rally("deployment check", print_output=True)
|
|
|
|
render_vars = {"verifications": []}
|
|
|
|
# Verification management stuff
|
|
render_vars["install"] = call_rally("verify install")
|
|
render_vars["genconfig"] = call_rally("verify genconfig")
|
|
render_vars["showconfig"] = call_rally("verify showconfig")
|
|
|
|
# Create a file with a list of tests that are expected to fail
|
|
xfails_file_path = create_file_with_xfails()
|
|
|
|
# Launch verification
|
|
launch_params = "%s --xfails-file %s" % (
|
|
MODES_PARAMETERS[args.mode], xfails_file_path)
|
|
render_vars["verifications"].append(
|
|
launch_verification_once(launch_params))
|
|
|
|
if args.compare:
|
|
render_vars["verifications"].append(
|
|
launch_verification_once(launch_params))
|
|
render_vars["compare"] = do_compare(
|
|
render_vars["verifications"][-2]["uuid"],
|
|
render_vars["verifications"][-1]["uuid"])
|
|
|
|
render_vars["list"] = call_rally("verify list")
|
|
|
|
render_page(**render_vars)
|
|
|
|
return _return_status
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|