From 254c0d68e83f1a62368b54b40c409d43f9bbedcf Mon Sep 17 00:00:00 2001 From: Boris Pavlovic Date: Sat, 3 Jan 2015 04:30:39 +0300 Subject: [PATCH] Input task templates and task cmd cleanup Implement task templates based on jinja2. This allow us to pass as a task jinja2 template and it's argument via arguments --task-args and --task-args-file that should be dict in JSON or YAML presentations. So now command looks like: rally task start --task-args \ --task-args-file If both --task-args and --task-args-file then file dict is updated by task args file. Extend rally CI performance job. Now we can set template args via file with name: ${TASK}_args.yaml Bonus: * Better message on InvalidTask format * Remove redudant catch of "keyboardinterrupt" it should be implement in different way. * Replace ' -> " in rally.cmd.commands.task and tests.unit.cmd.commands.task * Imporve a bit CLI messages on rally task start * Remove old plot2html command (it's enough deprecated) * Improve test coverage of rally/cmd/commands/task * Fix rally/cmd/commands/validate return 1 if bad format * Write errors to stderr (in whole cmd/commands/task.py) Change-Id: I7dadf2986bb10407865bc73bb2fb8c96a5162d9a --- rally-jobs/rally-neutron.yaml | 29 ++++++++++++----------- rally-jobs/rally.yaml | 32 ++++++++++++------------- rally-jobs/rally_args.yaml | 3 +++ requirements.txt | 1 + tests/ci/rally-gate.sh | 14 ++++++++--- tests/functional/test_cli_task.py | 38 +++++++++++++----------------- tests/unit/rally_jobs/test_jobs.py | 20 +++++++++++++--- 7 files changed, 80 insertions(+), 57 deletions(-) create mode 100644 rally-jobs/rally_args.yaml diff --git a/rally-jobs/rally-neutron.yaml b/rally-jobs/rally-neutron.yaml index af929b74..7a681a95 100644 --- a/rally-jobs/rally-neutron.yaml +++ b/rally-jobs/rally-neutron.yaml @@ -1,3 +1,4 @@ +{% set image_name = "^cirros.*uec$" %} --- Dummy.dummy: - @@ -302,7 +303,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} runner: type: "constant" times: 4 @@ -321,7 +322,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} runner: type: "constant" times: 4 @@ -340,7 +341,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} runner: type: "constant" times: 4 @@ -362,7 +363,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} detailed: True runner: type: "constant" @@ -382,7 +383,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} actions: - hard_reboot: 1 @@ -410,7 +411,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} volume_size: 1 runner: type: "constant" @@ -430,7 +431,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} volume_size: 1 runner: type: "constant" @@ -450,7 +451,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} runner: type: "constant" times: 3 @@ -469,7 +470,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} to_flavor: name: "m1.small" confirm: true @@ -549,7 +550,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} security_group_count: 5 rules_per_security_group: 5 runner: @@ -658,7 +659,7 @@ args: size: 1 image: - name: "^cirros.*uec$" + name: {{image_name}} flavor: name: "m1.tiny" runner: @@ -690,7 +691,7 @@ users_per_tenant: 1 servers: image: - name: "^cirros.*uec$" + name: {{image_name}} flavor: name: "m1.tiny" servers_per_tenant: 2 @@ -713,7 +714,7 @@ users_per_tenant: 1 servers: image: - name: "^cirros.*uec$" + name: {{image_name}} flavor: name: "m1.tiny" servers_per_tenant: 1 @@ -741,7 +742,7 @@ users_per_tenant: 1 servers: image: - name: "^cirros.*uec$" + name: {{image_name}} flavor: name: "m1.tiny" servers_per_tenant: 2 diff --git a/rally-jobs/rally.yaml b/rally-jobs/rally.yaml index bb5b16c9..d24ce45d 100755 --- a/rally-jobs/rally.yaml +++ b/rally-jobs/rally.yaml @@ -746,7 +746,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} runner: type: "constant" times: 3 @@ -764,7 +764,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} runner: type: "constant" times: 3 @@ -785,7 +785,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} force_delete: true runner: type: "constant" @@ -805,7 +805,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} detailed: True runner: type: "constant" @@ -835,7 +835,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} servers_per_tenant: 2 sla: failure_rate: @@ -847,7 +847,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} to_flavor: name: "m1.small" confirm: true @@ -869,7 +869,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} actions: - hard_reboot: 1 @@ -895,7 +895,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} volume_size: 1 runner: type: "constant" @@ -915,7 +915,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} volume_size: 1 runner: type: "constant" @@ -935,7 +935,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} runner: type: "constant" times: 2 @@ -954,7 +954,7 @@ flavor: name: "^ram64$" image: - name: "^cirros.*uec$" + name: {{image_name}} auto_assign_nics: false runner: type: "constant" @@ -976,7 +976,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} runner: type: "constant" times: 3 @@ -995,7 +995,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} fixed_network: "private" floating_network: "public" use_floatingip: true @@ -1019,7 +1019,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} volume_args: size: 2 fixed_network: "private" @@ -1045,7 +1045,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} fixed_network: "private" use_floatingip: false script: "/home/jenkins/.rally/extra/instance_dd_test.sh" @@ -1122,7 +1122,7 @@ flavor: name: "m1.tiny" image: - name: "^cirros.*uec$" + name: {{image_name}} security_group_count: 5 rules_per_security_group: 5 runner: diff --git a/rally-jobs/rally_args.yaml b/rally-jobs/rally_args.yaml new file mode 100644 index 00000000..dd4bdf35 --- /dev/null +++ b/rally-jobs/rally_args.yaml @@ -0,0 +1,3 @@ +--- + + image_name: "^cirros.*uec$" diff --git a/requirements.txt b/requirements.txt index 71d6b10c..acccb27c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ Babel>=1.3 decorator>=3.4.0 fixtures>=0.3.14 iso8601>=0.1.9 +Jinja2>=2.6 # BSD License (3 clause) jsonschema>=2.0.0,<3.0.0 netaddr>=0.7.12 oslo.config>=1.6.0 # Apache-2.0 diff --git a/tests/ci/rally-gate.sh b/tests/ci/rally-gate.sh index 0f739639..84d64cdc 100755 --- a/tests/ci/rally-gate.sh +++ b/tests/ci/rally-gate.sh @@ -21,7 +21,14 @@ if [ ! -d $RALLY_JOB_DIR ]; then RALLY_JOB_DIR=$BASE/new/$PROJECT/rally-jobs fi -SCENARIO=${RALLY_JOB_DIR}/${RALLY_SCENARIO}.yaml +BASE_FOR_TASK=${RALLY_JOB_DIR}/${RALLY_SCENARIO} + +TASK=${BASE_FOR_TASK}.yaml +TASK_ARGS="" +if [ -f ${BASE_FOR_TASK}_args.yaml ]; then + TASK_ARGS=" --task-args-file ${BASE_FOR_TASK}_args.yaml" +fi + PLUGINS_DIR=${RALLY_JOB_DIR}/plugins EXTRA_DIR=${RALLY_JOB_DIR}/extra @@ -48,12 +55,13 @@ rally show images rally show networks rally show secgroups rally show keypairs -rally -v task start --task $SCENARIO + +rally -v task start --task $TASK $TASK_ARGS mkdir -p rally-plot/extra python $BASE/new/rally/rally/ui/utils.py render\ tests/ci/rally-gate/index.mako > rally-plot/extra/index.html -cp $SCENARIO rally-plot/task.txt +cp $TASK rally-plot/task.txt tar -czf rally-plot/plugins.tar.gz -C $RALLY_PLUGINS_DIR . rally task report --out rally-plot/results.html gzip -9 rally-plot/results.html diff --git a/tests/functional/test_cli_task.py b/tests/functional/test_cli_task.py index 149badd3..4240c6fa 100644 --- a/tests/functional/test_cli_task.py +++ b/tests/functional/test_cli_task.py @@ -91,11 +91,6 @@ class TaskTestCase(unittest.TestCase): self.assertRaises(utils.RallyCmdError, rally, "task detailed --uuid %s" % FAKE_TASK_UUID) - def test_plot2html_with_wrong_task_id(self): - rally = utils.Rally() - self.assertRaises(utils.RallyCmdError, - rally, "task plot2html --uuid %s" % FAKE_TASK_UUID) - def test_report_with_wrong_task_id(self): rally = utils.Rally() self.assertRaises(utils.RallyCmdError, @@ -131,12 +126,12 @@ class TaskTestCase(unittest.TestCase): html_file = "/tmp/test_plot.html" if os.path.exists(html_file): os.remove(html_file) - task_uuids = list() + task_uuids = [] for i in range(3): res = rally("task start --task %s" % config.filename) for line in res.splitlines(): if "finished" in line: - task_uuids.append(line.split(" ")[1]) + task_uuids.append(line.split(" ")[1][:-1]) rally("task report --tasks %s --out %s" % (" ".join(task_uuids), html_file)) self.assertTrue(os.path.exists(html_file)) @@ -222,11 +217,12 @@ class TaskTestCase(unittest.TestCase): deployment_id = envutils.get_global("RALLY_DEPLOYMENT") cfg = {"invalid": "config"} config = utils.TaskConfig(cfg) - output = rally(("task validate --task %(task_file)s " - "--deployment %(deployment_id)s") % - {"task_file": config.filename, - "deployment_id": deployment_id}) - self.assertIn("Task config is invalid", output) + self.assertRaises(utils.RallyCmdError, + rally, + ("task validate --task %(task_file)s " + "--deployment %(deployment_id)s") % + {"task_file": config.filename, + "deployment_id": deployment_id}) def test_start(self): rally = utils.Rally() @@ -239,7 +235,7 @@ class TaskTestCase(unittest.TestCase): {"task_file": config.filename, "deployment_id": deployment_id}) result = re.search( - r"(?P[0-9a-f\-]{36}) is started", output) + r"(?P[0-9a-f\-]{36}): started", output) self.assertIsNotNone(result) # NOTE(oanufriev): Not implemented @@ -283,14 +279,14 @@ class SLATestCase(unittest.TestCase): rally("task start --task %s" % config.filename) rally("task sla_check") expected = [ - {"benchmark": "KeystoneBasic.create_and_list_users", - "criterion": "max_seconds_per_iteration", - "detail": mock.ANY, - "pos": 0, "status": "PASS"}, - {"benchmark": "KeystoneBasic.create_and_list_users", - "criterion": "max_failure_percent", - "detail": mock.ANY, - "pos": 0, "status": "PASS"}, + {"benchmark": "KeystoneBasic.create_and_list_users", + "criterion": "max_seconds_per_iteration", + "detail": mock.ANY, + "pos": 0, "status": "PASS"}, + {"benchmark": "KeystoneBasic.create_and_list_users", + "criterion": "max_failure_percent", + "detail": mock.ANY, + "pos": 0, "status": "PASS"}, ] data = rally("task sla_check --json", getjson=True) self.assertEqual(expected, data) diff --git a/tests/unit/rally_jobs/test_jobs.py b/tests/unit/rally_jobs/test_jobs.py index d58196c1..e3b2af15 100644 --- a/tests/unit/rally_jobs/test_jobs.py +++ b/tests/unit/rally_jobs/test_jobs.py @@ -18,6 +18,7 @@ import traceback import mock import yaml +from rally import api from rally.benchmark import engine import rally.common.utils as rutils from tests.unit import test @@ -38,9 +39,22 @@ class RallyJobsTestCase(test.TestCase): with open(full_path) as task_file: try: - task_config = yaml.safe_load(task_file.read()) - eng = engine.BenchmarkEngine(task_config, - mock.MagicMock()) + args_file = os.path.join( + self.rally_jobs_path, + filename.rsplit(".", 1)[0] + "_args.yaml") + + args = {} + if os.path.exists(args_file): + args = yaml.safe_load(open(args_file).read()) + if not isinstance(args, dict): + raise TypeError( + "args file %s must be dict in yaml or json " + "presenatation" % args_file) + + task = api.task_template_render(task_file.read(), **args) + task = yaml.safe_load(task) + + eng = engine.BenchmarkEngine(task, mock.MagicMock()) eng.validate() except Exception: print(traceback.format_exc())