diff --git a/doc/source/images/Report-Verify-filter-by-status.png b/doc/source/images/Report-Verify-filter-by-status.png new file mode 100644 index 00000000..9088696d Binary files /dev/null and b/doc/source/images/Report-Verify-filter-by-status.png differ diff --git a/doc/source/images/Report-Verify-for-4-Verifications.png b/doc/source/images/Report-Verify-for-4-Verifications.png new file mode 100644 index 00000000..771beb3d Binary files /dev/null and b/doc/source/images/Report-Verify-for-4-Verifications.png differ diff --git a/doc/source/images/Report-Verify-toggle-tags.png b/doc/source/images/Report-Verify-toggle-tags.png new file mode 100644 index 00000000..a5694095 Binary files /dev/null and b/doc/source/images/Report-Verify-toggle-tags.png differ diff --git a/doc/source/images/Report-Verify-tracebacks.png b/doc/source/images/Report-Verify-tracebacks.png new file mode 100644 index 00000000..66e96191 Binary files /dev/null and b/doc/source/images/Report-Verify-tracebacks.png differ diff --git a/doc/source/overview/glossary.rst b/doc/source/overview/glossary.rst index 4e289a70..69491d25 100644 --- a/doc/source/overview/glossary.rst +++ b/doc/source/overview/glossary.rst @@ -1,5 +1,7 @@ :tocdepth: 1 +.. _glossary: + ======== Glossary ======== @@ -165,6 +167,8 @@ Rally can run different subunit-based testing tools against a target environment, for example `tempest `_ for OpenStack. +.. _glossary-verification: + Verification ------------ diff --git a/doc/source/verification/cli_reference.rst b/doc/source/verification/cli_reference.rst new file mode 100644 index 00000000..535609f7 --- /dev/null +++ b/doc/source/verification/cli_reference.rst @@ -0,0 +1,27 @@ +.. + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _rally-verify-cli-reference: + +====================== +Command Line Interface +====================== + +Cut down from Global :ref:`cli-reference` + +.. contents:: + :depth: 2 + :local: + +.. make_cli_reference:: + :group: verify diff --git a/doc/source/verification/howto/add_new_reporter.rst b/doc/source/verification/howto/add_new_reporter.rst new file mode 100644 index 00000000..d16a4a50 --- /dev/null +++ b/doc/source/verification/howto/add_new_reporter.rst @@ -0,0 +1,104 @@ +.. + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _howto-add-new-reporting-mechanism: + +================================= +HowTo add new reporting mechanism +================================= + +Reporting mechanism for verifications is pluggable. Custom plugins can be used +for custom output formats or for exporting results to external systems. + +We hardly recommend to read :ref:`plugins` page to understand how do Rally +Plugins work. + +.. contents:: + :depth: 2 + :local: + +Spec +---- + +All reporters should inherit +``rally.verification.reporter.VerificationReporter`` and implement all +abstract methods. Here you can find its interface: + + .. autoclass:: rally.verification.reporter.VerificationReporter + :members: + +Example of custom JSON Reporter +------------------------------- + +Basically, you need to implement only two methods "validate" and "generate". + +Method "validate" should check that destination of the report is right. +Method "generate" should build a report or export results somewhere; actually, +it is up to you what it should do but return format is strict, see :ref:`spec` +section for what it can return. + +.. code-block:: python + + import json + + from rally.verification import reporter + + + @reporter.configure("summary-in-json") + class SummaryInJsonReporter(reporter.VerificationReporter): + """Store summary of verification(s) in JSON format""" + + # ISO 8601 + TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z" + + @classmethod + def validate(cls, output_destination): + # we do not have any restrictions for destination, so nothing to + # check + pass + + def generate(self): + report = {} + + for v in self.verifications: + report[v.uuid] = { + "started_at": v.created_at.strftime(self.TIME_FORMAT), + "finished_at": v.updated_at.strftime(self.TIME_FORMAT), + "status": v.status, + "run_args": v.run_args, + "tests_count": v.tests_count, + "tests_duration": v.tests_duration, + "skipped": v.skipped, + "success": v.success, + "expected_failures": v.expected_failures, + "unexpected_success": v.unexpected_success, + "failures": v.failures, + # v.tests includes all information about launched tests, + # but for simplification of this fake reporters, let's + # save just names + "launched_tests": [test["name"] + for test in v.tests.values()] + } + + raw_report = json.dumps(report, indent=4) + + if self.output_destination: + # In case of output_destination existence report will be saved + # to hard drive and there is nothing to print to stdout, so + # "print" key is not used + return {"files": {self.output_destination: raw_report}, + "open": self.output_destination} + else: + # it is something that will be print at CLI layer. + return {"print": raw_report} + diff --git a/doc/source/verification/howto/add_support_for_new_tool.rst b/doc/source/verification/howto/add_support_for_new_tool.rst new file mode 100644 index 00000000..2352c895 --- /dev/null +++ b/doc/source/verification/howto/add_support_for_new_tool.rst @@ -0,0 +1,117 @@ +.. + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _howto-add-support-for-new-tool: + +============================== +HowTo add support for new tool +============================== + +First of all, you should start from the reading of :ref:`plugins` page. +After you learned basic things about Rally plugin mechanism, let's move to +Verifier interface itself. + +.. contents:: + :depth: 2 + :local: + +Spec +---- + +All verifiers plugins should inherit +``rally.verification.manager.VerifierManager`` and implement all abstract +methods. Here you can find its interface: + + .. autoclass:: rally.verification.manager.VerifierManager + :members: + :exclude-members: base_ref, check_system_wide, checkout, install_venv, + parse_results, validate + + +Example of Fake Verifier Manager +-------------------------------- + +FakeTool is a tool which doesn't require configuration and installation. + + .. code-block:: python + + import random + import re + + from rally.verification import manager + + + # Verification component expects that method "run" of verifier returns + # object. Class Result is a simple wrapper for two expected properties. + class Result(object): + def __init__(self, totals, tests): + self.totals = totals + self.tests = tests + + + @manager.configure("fake-tool", default_repo="https://example.com") + class FakeTool(manager.VerifierManager): + """Fake Tool \o/""" + + TESTS = ["fake_tool.tests.bar.FatalityTestCase.test_one", + "fake_tool.tests.bar.FatalityTestCase.test_two", + "fake_tool.tests.bar.FatalityTestCase.test_three", + "fake_tool.tests.bar.FatalityTestCase.test_four", + "fake_tool.tests.foo.MegaTestCase.test_one", + "fake_tool.tests.foo.MegaTestCase.test_two", + "fake_tool.tests.foo.MegaTestCase.test_three", + "fake_tool.tests.foo.MegaTestCase.test_four"] + + # This fake verifier doesn't launch anything, just returns random + # results, so let's override parent methods to avoid redundant + # clonning repo, checking packages and so on. + + def install(self): + pass + + def uninstall(self, full=False): + pass + + # Each tool, which supports configuration, has the own mechanism + # for that task. Writing unified method is impossible. That is why + # `VerificationManager` implements the case when the tool doesn't + # need (doesn't support) configuration at all. Such behaviour is + # ideal for FakeTool, since we do not need to change anything :) + + # Let's implement method `run` to return random data. + def run(self, context): + + totals = {"tests_count": len(self.TESTS), + "tests_duration": 0, + "failures": 0, + "skipped": 0, + "success": 0, + "unexpected_success": 0, + "expected_failures": 0} + tests = {} + for name in self.TESTS: + duration = random.randint(0, 10000)/100. + totals["tests_duration"] += duration + test = {"name": name, + "status": random.choice(["success", "fail"]), + "duration": "%s" % duration} + if test["status"] == "fail": + test["traceback"] = "Ooooppps" + totals["failures"] += 1 + else: + totals["success"] += 1 + tests[name] = test + return Result(totals, tests=tests) + + def list_tests(self, pattern=""): + return [name for name in self.TESTS if re.match(pattern, name)] diff --git a/doc/source/verification/howto/index.rst b/doc/source/verification/howto/index.rst new file mode 100644 index 00000000..e2b68a4b --- /dev/null +++ b/doc/source/verification/howto/index.rst @@ -0,0 +1,12 @@ + +===== +HowTo +===== + +.. toctree:: + :maxdepth: 1 + :glob: + + ./add* + migrate_from_old_design + diff --git a/doc/source/verification/howto/migrate_from_old_design.rst b/doc/source/verification/howto/migrate_from_old_design.rst new file mode 100644 index 00000000..4a7ffaef --- /dev/null +++ b/doc/source/verification/howto/migrate_from_old_design.rst @@ -0,0 +1,510 @@ +.. + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +======================================================== +HowTo migrate from Verification component 0.7.0 to 0.8.0 +======================================================== + +.. note:: This document describes migration process from 0.7.0 to 0.8.0 Rally + version. You can apply this instruction for migration to later versions, + but check all references and release notes before trying to do it. + + +Verification Component was introduced long time ago even before the first Rally +release. It started as a small helper thing but became a big powerful tool. +Since it was not designed to all features that were implemented there later, +it contained a lot of workarounds and hacks. + +New Verification Component, which we are happy to introduce, should fix all +architecture issues and improve user-experience. Unfortunately, fixing all +those obsolete architecture decisions could not be done in a +backward-compatible way, or it would produce much more workarounds. That is why +we decided to redesign the whole component in a clear way - remove old code and +write a new one from scratch. + +Migration to New Verification Component should be simple and do not take too +much time. You can find description of made changes below. + +.. contents:: + :depth: 2 + :local: + +Reports +------- + +We completely reworked verification reports and merged comparison to main +report. Now you can build one report for multiple number of verifications. + +For more details follow :ref:`verification-reports` + +Verification statuses +--------------------- + ++------------+------------+---------------------------------------------------+ +| Old Status | New Status | Description | ++============+============+===================================================+ +| init | init | Initial state. It appears instantly after calling | +| | | ``rally verify start`` command before the actual | +| | | run of verifier's tool. | ++------------+------------+---------------------------------------------------+ +| running | | It was used right after checking status of | +| | | verifier. It is redundant in terms of new design. | ++------------+------------+---------------------------------------------------+ +| verifying | running | Identifies the process of execution tool. | ++------------+------------+---------------------------------------------------+ +| finished | finished | Previously, finished state was used for | +| | | identification of just finished verification. By | +| | | "finished" meant that verification has any test | +| | | result. Now it means that verification was | +| | | executed and doesn't have failures, unexpected | +| | | success or any kind of errors. | +| +------------+---------------------------------------------------+ +| | failed | Old purpose is identification "errors", situations| +| | | when results are empty. The right use is | +| | | an identification finished verification with | +| | | tests in failed and unexpected success statuses. | ++------------+------------+---------------------------------------------------+ +| failed | crashed | Something went wrong while launching verification.| ++------------+------------+---------------------------------------------------+ + +Latest information about verification statuses you can find at +:ref:`verification_statuses`. + +Command Line Interface +---------------------- + +You can find the latest information about Verification Component CLI here - +:ref:`rally-verify-cli-reference`. + +Installing verifier +""""""""""""""""""" + +Command for Rally 0.7.0 - `rally verify install +`_ + +.. code-block:: console + + $ rally verify install --deployment --source --version \ + --system-wide + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify create-verifier --type "tempest" --source \ + --version --system-wide + +Here you can find several important improvements: + +1) Rally team introduced new entity - :ref:`verifiers`. Verifier stores all + information about installed tool (i.e., source, version, system-wide) in a + database. You do not need to transmit the same arguments into + all ``rally verify`` commands as it was previously with ``--system-wide`` + flag. + +2) You can use particular verifier for multiple deployments. ``--deployment`` + flag moved to ``rally verify start`` command. Also, you can run it + simultaneously (checking in parallel different sets, different cloud, etc) + +3) Verification Component can use not only Tempest for verifying system. Check + :ref:`known-verifier-types` for full list of supported tools. + +4) You can have unlimited number of verifiers. + +Re-install verifier aka update +"""""""""""""""""""""""""""""" + +Command for Rally 0.7.0 - `rally verify reinstall +`_ + +.. code-block:: console + + $ rally verify reinstall --deployment --source --version \ + --system-wide + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify update-verifier --id --source --version \ + --system-wide --no-system-wide --update-venv + +Changes: + +1) ``rally verify update-verifier`` doesn't require deployment id + +2) You can switch between usage of system-wide installation and virtual + environment. + +3) You can update just virtual environment without cloning verifier code again + +Uninstall +""""""""" + +Command for Rally 0.7.0 - `rally verify uninstall +`_ + +.. code-block:: console + + $ rally verify uninstall --deployment + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify delete-verifier --id --deployment-id --force + +Changes: + +1) As it was mentioned before, Verifier doesn't have an alignment to any + particular deployment, so deployment argument is optional now. + If --deployment-id argument is specified only deployment specific data will + be removed (i.e, configurations). + +2) New --force flag for removing all verifications results for that verifier. + +Installation extensions +""""""""""""""""""""""" + +Command for Rally 0.7.0 - `rally verify installplugin +`_ + +.. code-block:: console + + $ rally verify installplugin --deployment --source \ + --version --system-wide + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify add-verifier-ext --id --source --version \ + --extra-settings + +Changes: + +1) --system-wide flag is removed. Rally checks the verifier information to + identify where to install the extension - in a system-side way or use + virtual environment. + +2) New --extra-settings flag. In case of Tempest, it is redundant, but for + other verifiers allows to transmit some extra installation settings for + verifier extension. + +Uninstall extensions +"""""""""""""""""""" + +Command for Rally 0.7.0 - `rally verify uninstallplugin +`_ + + +.. code-block:: console + + $ rally verify uninstallplugin --deployment --repo-name \ + --system-wide + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify delete-verifier-ext --id --name + +Changes: + +1) It is one more place where you do not need to pass --system-wide flag + anymore. + +2) --deployment flag is gone. + +3) --repo-name is renamed to just --name. + +List extensions +""""""""""""""" + +Command for Rally 0.7.0 - `rally verify listplugins +`_ + +.. code-block:: console + + $ rally verify listplugins --deployment --system-wide + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify list-verifier-exts --id + +Changes: + +1) No need to specify --system-wide flag. + +2) --deployment flag is gone. + +Discover available tests +"""""""""""""""""""""""" + +Command for Rally 0.7.0 - `rally verify discover +`_ + +.. code-block:: console + + $ rally verify discover --deployment --system-wide --pattern + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify list-verifier-tests --id --pattern + +Changes: + +1) No need to specify --system-wide flag. + +2) --deployment flag is gone. + +Configuring +""""""""""" + +Commands for Rally 0.7.0: + +* The command for generating configs `rally verify genconfig + `_ + + .. code-block:: console + + $ rally verify genconfig --deployment --tempest-config \ + --add-options --override + +* The command for showing configs `rally verify showconfig + `_ + + .. code-block:: console + + $ rally verify showconfig --deployment + + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify configure-verifier --id --deployment-id \ + --extend --override --reconfigure --show + +Changes: + +1) The argument ``--override`` replaces old ``--tempest-config`` name. First + of all, argument name "override" is a unified word without alignment to any + tool. Also, it describes in the best way the meaning of the action: use + client specified configuration file. + +2) The argument ``--extend`` replaces old ``--add-options``. It accepts a path + to config in INI format or JSON/YAML string. In future, it will be extended + with the ability to specify a path to JSON/YAML file. + +3) The argument ``--reconfigure`` replaces old ``--override``. It means that + existing file will be ignored and new one will be used/created. + +4) If the argument ``--show`` is specified, a configuration of verifier will be + displayed at the end of command execution. + + .. note:: We do not have a separate command for showing configurations + anymore. ``rally verify configure-verifier --show`` shows an existing + configuration if it exists and ``--reconfigure`` argument is not specified + +Show config +""""""""""" + +Command for Rally 0.7.0 - `rally verify showconfig +`_ + +.. code-block:: console + + $ rally verify showconfig --deployment + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify configure-verifier --id --deployment-id --show + +Changes: + + We do not have a separate command for that task. + ``rally verify configure-verifier --show`` shows an existing configuration + (if it exists) in case of not specified ``--reconfigure`` argument. + +Running verification +"""""""""""""""""""" + +Command for Rally 0.7.0 - `rally verify start +`_ + +.. code-block:: console + + $ rally verify start --deployment --set --regex \ + --load-list --tests-file --skip-list \ + --tempest-config --xfail-list --system-wide \ + --concurrency --failing --no-use + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify start --id --deployment-id --pattern \ + --load-list --skip-list --xfail-list \ + --concurrency --no-use + +Changes: + +1) You need to pass verifier id + +2) Arguments ``--set`` and ``--regex`` are merged in the new model to single + ``--pattern`` argument. Name of tests set should be specified like + ``--pattern set_name=``. It was done to provide a way for each + verifier to support custom arguments. + +3) The argument ``--tests-file`` was deprecated in Rally 0.6.0 and + time&ability come to delete it. +4) Arguments ``--skip-list`` and ``--xfail-list`` accept path to file in + JSON/YAML format. Content should be a dictionary, where keys are tests + names (full name with id and tags) and values are reasons. +5) The argument ``--tempest-config`` is gone. Use + ``rally verify configure-verifier --id --deployment-id + --override `` instead. +6) The argument ``--system-wide`` is gone like in most of other commands. + +Show verification result +"""""""""""""""""""""""" + +Commands for Rally 0.7.0: + +* The command for showing results of verification `rally verify show + `_ + + .. code-block:: console + + $ rally verify show --uuid --sort-by --detailed + +* Separate command witch call ``rally verify show`` with hardcoded + ``--detailed`` flag `rally verify showconfig + `_ + + .. code-block:: console + + $ rally verify detailed --uuid --sort-by + + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify show --uuid --sort-by --detailed + +Changes: + +1) Redundant ``rally verify detailed`` command is removed + +2) Sorting tests via ``--sort-by`` argument is extended to name/duration/status + +Listing all verifications +""""""""""""""""""""""""" + +Command for Rally 0.7.0 - `rally verify list +`_ + +.. code-block:: console + + $ rally verify list + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify list --id --deployment-id --status + +Changes: + + You can filter verifications by verifiers, by deployments and by results + statuses. + +Importing results +""""""""""""""""" + +Command for Rally 0.7.0 - `rally verify import +`_ + +.. code-block:: console + + $ rally verify import --deployment --set --file --no-use + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify import --id --deployment --file \ + --run-args --no-use + +Changes: + +1) You need to specify verifier to import results for. + +2) The argument ``--set`` is merged into unified ``--run-args``. + +Building reports +"""""""""""""""" + +Commands for Rally 0.7.0: + +* The command for building HTML/JSON reports of verification + `rally verify results + `_ + + .. code-block:: console + + $ rally verify results --uuid --html --json --output-file + +* The command for comparison two verifications `rally verify compare + `_ + + .. code-block:: console + + $ rally verify compare --uuid-1 --uuid-2 --csv --html \ + --json --output-file --threshold + +Command for Rally 0.8.0: + +.. code-block:: console + + $ rally verify report --uuid --type --to --open + +Changes: + +1) Building reports becomes pluggable. You can extend reporters types. + See :ref:`verification-reports` for more details. + +2) The argument ``--type`` expects type of report (HTML/JSON). There are no + more separate arguments for each report type. + +3) Reports are not aligned to only local types, so the argument ``--to`` + replaces ``--output-file``. In case of HTML/JSON reports, it can include a + path to the local file like it was previously or URL to some external system + with credentials like ``https://username:password@example.com:777``. + +4) The comparison is embedded into main reports and it is now limited by two + verifications results. There are no reasons for the separate command for + that task. + +The End +""""""" + +Have nice verifications! diff --git a/doc/source/verification/index.rst b/doc/source/verification/index.rst index 27d32e45..3b2254e4 100644 --- a/doc/source/verification/index.rst +++ b/doc/source/verification/index.rst @@ -23,5 +23,12 @@ environment works by different tools (we call them :ref:`glossary-verification`). .. toctree:: - :maxdepth: 3 + :maxdepth: 2 :glob: + + verifiers + reports + cli_reference + howto/index + +.. include:: ./overview.rst diff --git a/doc/source/verification/overview.rst b/doc/source/verification/overview.rst new file mode 100644 index 00000000..0ca8a0df --- /dev/null +++ b/doc/source/verification/overview.rst @@ -0,0 +1,57 @@ +.. + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Historical background +--------------------- + +Tempest, OpenStack’s official test suite, is a powerful tool for running a set +of functional tests against an OpenStack cluster. Tempest automatically runs +against every patch in every project of OpenStack, which lets us avoid merging +changes that break functionality. + +Unfortunately, it has limited opportunities to be used, to process its results, +etc. That is why we started Verification Component initiative a long time ago +(see `a blog post +`_ +for more details, but be careful as all user interface is changed completely +since that time). + +What is Verification Component and why do you need it? +------------------------------------------------------ + +The primary goal of Rally Product is to provide a simple way to do complex +things. As for functional testing, Verification component includes interfaces +for: + +* **Managing things**. Create an isolated virtual environment and install + verification tool there? Yes, we can do it! Clone tool from Git repositories? + Sure! Store several versions of one tool (you know, sometimes they are + incompatible, with different required packages and so on)? Of course! + In general, Verification Component allows to install, upgrade, reinstall, + configure your tool. You should not care about zillion options anymore Rally + will discover them via cloud UX and make the configuration file for you + automatically. +* **Launching verifiers**. Launchers of specific tools don't always contain all + required features, Rally team tries to fix this omission. Verification + Component support some of them like expected failures, the list of tests to + skip, a list of tests to launch, re-running previous verification or just + failed tests from it and so on. Btw, all verification runs arguments are + stored in the database. +* **Processing results**. Rally DataBase stores all `verifications + `_ and you can obtain unified (across different verifiers) + results at any time. You can find a verification run summary there, run + arguments which were used, error messages and etc. Comparison mechanism for + several verifications is available too. Verification reports can be generated + in several formats: HTML, JSON, JUnit-XML (see :ref:`verification-reports` + for more details). Also, reports mechanism is expendable and you can write + your own plugin for whatever system you want. diff --git a/doc/source/verification/reports.rst b/doc/source/verification/reports.rst new file mode 100644 index 00000000..419c0419 --- /dev/null +++ b/doc/source/verification/reports.rst @@ -0,0 +1,114 @@ +.. + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _verification-reports: + +==================== +Verification reports +==================== + +Rally stores all verifications results in its DataBase so that you can access +and process results at any time. No matter what verifier you use, results will +be stored in a unified way and reports will be unified too. + +Out of the box, we support several types of reports out of the +box: :include-var:`rally.cli.commands.verify.DEFAULT_REPORT_TYPES`; but our +reporting system is pluggable so that you can write your own plugin to build +some specific reports or to export results to the specific system (see +:ref:`howto-add-new-reporting-mechanism` for more details`). + +.. contents:: + :depth: 2 + :local: + +HTML reports +------------ + +HTML report is the most convenient type of reports. It includes as much as +possible useful information about Verifications. + +Here is an example of HTML report for 3 verifications. +It was generated by next command: + +.. code-block:: console + + $ rally verify report --uuid --type html \ + --to ./report.html + + +.. image:: ../images/Report-Verify-for-4-Verifications.png + :align: center + +The report consists of two tables. + +First one is a summary table. It includes base information about +verifications: UUIDs; numbers of tests; when they were launched; statuses; etc. +Also, you can find detailed information grouped by tests statuses at the right +part of the table. + +If the size (height) of the summary table seems too large for you and hinders +to see more tests results, you can push "Toggle Header" button. + +The second table contains actual verifications results. They are grouped by +tests names. The result of the test for particular verification overpainted by +one of the next colours: + +* *Red* - It means that test has "failed" status +* *Orange* - It is "unexpected success". Most of the parsers calculates it just + like failure +* *Green* - Everything is ok. The test succeeded. +* *Yellow* - It is "expected failure". +* *Light Blue* - Test is skipped. It is nor good neither bad. + +Several verifications comparison is a default embedded behaviour of reports. +The difference between verifications is displayed in brackets after actual +test duration. Sign **+** means that current result is bigger that standard by +the number going after the sign. Sign **-** is an opposite to **-**. Please, +note that all diffs are comparisons with the first verification in a row. + +Filtering results +""""""""""""""""" + +You can filter tests by setting or removing a mark from check box of the +particular status column of the summary table. + +.. image:: ../images/Report-Verify-filter-by-status.png + :align: center + +Tests Tags +"""""""""" + +Some of the tests tools support tests tagging. It can be used for setting +unique IDs, groups, etc. Usually, such tags are included in test name. It is +inconvenient and Rally stores tags separately. By default they are hidden, but +if you push "Toggle tags" button, they will be displayed under tests names. + +.. image:: ../images/Report-Verify-toggle-tags.png + :align: center + +Tracebacks & Reasons +"""""""""""""""""""" + +Tests with "failed" and "expected failure" statuses have tracebacks of +failures. Tests with "skipped", "expected failure", "unexpected success" status +has "reason" of events. By default, both tracebacks and reasons are hidden, +but you can show them by clicking to the appropriate test. + +.. image:: ../images/Report-Verify-tracebacks.png + :align: center + +Plugins Reference for all out-of-the-box reporters +-------------------------------------------------- + +.. generate_plugin_reference:: + :base_cls: Verification Reporter diff --git a/doc/source/verification/verifiers.rst b/doc/source/verification/verifiers.rst new file mode 100644 index 00000000..392c72c3 --- /dev/null +++ b/doc/source/verification/verifiers.rst @@ -0,0 +1,100 @@ +.. + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +.. _verifiers: + +========= +Verifiers +========= + +.. contents:: + :depth: 1 + :local: + +What is it? +----------- + +Verifier Plugin is a compatibility layer between Rally and the specific tool +(such as Tempest) which runs tests. It implements features like installation, +configuration, upgrades, running, etc in terms of the tool. It is a driver in +other words. +It is a pluggable entity, which means that you can easily add support for +whatever tool you want (see :ref:`howto-add-support-for-new-tool` page for +more information). Even more, you can deliver such plugin separately from Rally +itself, but we firmly recommend to push a change to Rally upstream (see +:ref:`contribute` guide), so Rally core-team will able to review it and help +to improve. + +Verifier is an instance of the Verifier Plugin. It is an installed tool. +For example, "Tempest" is a set of functional tests, it is Verifier Plugin +(we have a plugin for it). Installed Tempest 12.0 from +https://github.com/openstack/tempest in a virtual environment is the verifier. + +Verifier is not aligned to any particular deployment like it was in the past, +you can use one verifier for testing unlimited number of deployments (each +deployment will have separate configuration files for the tool). + +Verifier & Verifier Plugin are the main entities which Verification component +operates with. Another one is the verifications results. + +Verifier statuses +----------------- + +All verifiers can be in next statuses: + +* *init* - Initial state. It appears while you call ``rally verify + create-verifier`` command and installation step is not yet started. +* *installing* - Installation of the verifier is not a quick task. It is about + cloning tool, checking packages or installing virtual environments with all + required packages. This state indicates that this step is in the process. +* *installed* - It should be one of your favourite states. It means that + everything is ok and you can start verifying your cloud. +* *updating* - This state identifies the process of updating verifier (version, + source, packages, etc.). +* *extending* - The process of extending a verifier by its plugins. +* *failed* - Something went wrong while installation. + +.. _verification_statuses: + +Verification statuses +--------------------- + +* *init* - Initial state. It appears instantly after calling + ``rally verify start`` command before the actual run of verifier's tool. +* *running* - Identifies the process of execution tool. +* *finished*- Verification is finished without errors and failures. +* *failed* - Verification is finished, but there are some failed tests. +* *crashed* - Unexpected error had happened while running verification. + + +.. _known-verifier-types: + +Known verifier types +-------------------- + +Out of the box +"""""""""""""" + +You can execute command ``rally verify list-plugins`` locally to check +available verifiers in your environment. + +Cut down from Global :ref:`plugin-reference` page: + +.. generate_plugin_reference:: + :base_cls: Verifier Manager + +Third-party +""""""""""" + +Nothing here yet. + diff --git a/rally/consts.py b/rally/consts.py index a61d0cdf..36eefa8e 100644 --- a/rally/consts.py +++ b/rally/consts.py @@ -202,6 +202,8 @@ class _VerifierStatus(utils.ImmutableMixin, utils.EnumMixin): FAILED = "failed" +# NOTE(andreykurilin): In case of updating these statuses, please do not forget +# to update doc reference too class _VerificationStatus(utils.ImmutableMixin, utils.EnumMixin): """Verification statuses.""" INIT = "init" diff --git a/rally/plugins/openstack/verification/tempest/manager.py b/rally/plugins/openstack/verification/tempest/manager.py index ec26e314..6553b081 100644 --- a/rally/plugins/openstack/verification/tempest/manager.py +++ b/rally/plugins/openstack/verification/tempest/manager.py @@ -30,13 +30,45 @@ from rally.verification import utils LOG = logging.getLogger(__name__) +AVAILABLE_SETS = (list(consts.TempestTestSets) + + list(consts.TempestApiTestSets) + + list(consts.TempestScenarioTestSets)) @manager.configure(name="tempest", namespace="openstack", default_repo="https://git.openstack.org/openstack/tempest", context={"tempest_configuration": {}, "testr_verifier": {}}) class TempestManager(testr.TestrLauncher): - """Plugin for Tempest management.""" + """Tempest verifier. + + **Description**: + + Quote from official documentation: + + This is a set of integration tests to be run against a live OpenStack + cluster. Tempest has batteries of tests for OpenStack API validation, + Scenarios, and other specific tests useful in validating an OpenStack + deployment. + + Rally supports features listed below: + + * *cloning Tempest*: repository and version can be specified + * *installation*: system-wide with checking existence of required + packages or in virtual environment + * *configuration*: options are discovered via OpenStack API, but you can + override them if you need + * *running*: pre-creating all required resources(i.e images, tenants, + etc), prepare arguments, launching Tempest, live-progress output + * *results*: all verifications are stored in db, you can built reports, + compare verification at whatever you want time. + + Appeared in Rally 0.8.0 *(actually, it appeared long time ago with first + revision of Verification Component, but 0.8.0 is mentioned since it is + first release after Verification Component redesign)* + """ + + RUN_ARGS = {"set_name": "Name of predefined sets of tests. Known names: " + "%s" % ", ".join(AVAILABLE_SETS)} @property def run_environ(self): @@ -158,14 +190,11 @@ class TempestManager(testr.TestrLauncher): if len(pattern) == 1: pass # it is just a regex elif pattern[0] == "set": - available_sets = (list(consts.TempestTestSets) + - list(consts.TempestApiTestSets) + - list(consts.TempestScenarioTestSets)) - if pattern[1] not in available_sets: + if pattern[1] not in AVAILABLE_SETS: raise exceptions.ValidationError( "Test set '%s' not found in available " "Tempest test sets. Available sets are '%s'." - % (pattern[1], "', '".join(available_sets))) + % (pattern[1], "', '".join(AVAILABLE_SETS))) else: raise exceptions.ValidationError( "'pattern' argument should be a regexp or set name "