From eef45465ca0ef2d6177621b4cd4622ac775cf5e1 Mon Sep 17 00:00:00 2001 From: Andrey Kurilin Date: Fri, 5 Dec 2014 01:19:36 +0200 Subject: [PATCH] Configure gate-rally-dsvm-verify Recently, we added new job "gate-rally-dsvm-verify" for Rally[1]. This job implied functional testing integration Rally with Tempest[2]. This patch adds tests/ci/rally-verify.sh script, which is used by new job. rally-verify.sh does: - tempest installation - run "rally verify start" twice and print results - compare results of two verifications - list verifications - generate html page based on results To implement gate-rally-dsvm-verify some changes were requered to existing Rally code: - Added ability for rally/ui/utils.py to accept arguments to render html-pages - Fixed logging debug-messages in tempest verifier - Fixed check "is debug mode turned on or not"(also, added hacking rule for it) TODO for future patches: - add launch of rally task for Tempest - add launch of random test set - add check for successful tests [1] https://review.openstack.org/#/c/137232 [2] https://www.mirantis.com/blog/rally-openstack-tempest-testing-made-simpler Closes-Bug: #1400465 Closes-Bug: #1400518 Change-Id: I8e1fbab22c2da109bbc442f040fe259e5d22a62a --- tests/functional/test_cli_verify.py | 44 +++++++++-------------------- tests/hacking/README.rst | 1 + tests/hacking/checks.py | 19 +++++++++++++ tests/unit/test_hacking.py | 10 +++++++ 4 files changed, 43 insertions(+), 31 deletions(-) diff --git a/tests/functional/test_cli_verify.py b/tests/functional/test_cli_verify.py index 631df542..903c1c75 100644 --- a/tests/functional/test_cli_verify.py +++ b/tests/functional/test_cli_verify.py @@ -13,35 +13,17 @@ # License for the specific language governing permissions and limitations # under the License. -import json -import unittest +############################################################################## +# +# THIS MODULE IS DEPRECATED. +# DON'T ADD TESTS FOR "rally verify" HERE. +# +# This module is no longer used for testing "rally verify" command. +# Functional testing for this command is moved to separate job. +# https://review.openstack.org/#/c/137232 +# +# Please look at tests/ci/rally-verify.sh for more details. +# +############################################################################## -from tests.functional import utils - - -class VerifyTestCase(unittest.TestCase): - - def setUp(self): - super(VerifyTestCase, self).setUp() - self.rally = utils.Rally() - - def _verify_start_and_get_results_in_json(self, set_name): - self.rally("verify start %s" % set_name) - results = json.loads(self.rally("verify results --json")) - - failed_tests = results["failures"] * 100.0 / results["tests"] - if failed_tests >= 50: - self.fail("Number of failed tests more than 50%.") - - show_output = self.rally("verify show") - - total_raw = show_output.split("\n").pop(5)[1:-1].replace(" ", "") - total = total_raw.split('|') - - self.assertEqual(set_name, total[2]) - self.assertEqual(results["tests"], int(total[3])) - self.assertEqual(results["failures"], int(total[4])) - self.assertEqual("finished", total[6]) - - def test_image_set(self): - self._verify_start_and_get_results_in_json("image") +pass diff --git a/tests/hacking/README.rst b/tests/hacking/README.rst index 256811fd..1a1d4881 100644 --- a/tests/hacking/README.rst +++ b/tests/hacking/README.rst @@ -14,6 +14,7 @@ Rally Specific Commandments * [N310-N314] - Reserved for rules related to logging * [N310] - Ensure that ``rally.log`` is used instead of ``rally.openstack.common.log`` * [N311] - Validate that debug level logs are not translated + * [N312] - Validate correctness of debug on check. * [N32x] - Reserved for rules related to assert* methods * [N320] - Ensure that ``assertTrue(isinstance(A, B))`` is not used * [N321] - Ensure that ``assertEqual(type(A), B)`` is not used diff --git a/tests/hacking/checks.py b/tests/hacking/checks.py index 20df5468..6f25d048 100644 --- a/tests/hacking/checks.py +++ b/tests/hacking/checks.py @@ -138,6 +138,24 @@ def no_translate_debug_logs(logical_line): yield(0, "N311 Don't translate debug level logs") +def no_use_conf_debug_check(logical_line, filename): + """Check for 'cfg.CONF.debug' + + Rally has two DEBUG level: + - Full DEBUG, which include all debug-messages from all OpenStack services + - Rally DEBUG, which include only Rally debug-messages + so we should use custom check to know debug-mode, instead of CONF.debug + + N312 + """ + excluded_files = ["./rally/log.py"] + + point = logical_line.find("CONF.debug") + if point != -1 and filename not in excluded_files: + yield(point, "N312 Don't use `CONF.debug`. " + "Function `rally.log.is_debug` should be used instead.") + + def assert_true_instance(logical_line): """Check for assertTrue(isinstance(a, b)) sentences @@ -209,6 +227,7 @@ def factory(register): register(check_assert_methods_from_mock) register(check_import_of_logging) register(no_translate_debug_logs) + register(no_use_conf_debug_check) register(assert_true_instance) register(assert_equal_type) register(assert_equal_none) diff --git a/tests/unit/test_hacking.py b/tests/unit/test_hacking.py index 75e053d4..f922eca0 100644 --- a/tests/unit/test_hacking.py +++ b/tests/unit/test_hacking.py @@ -93,6 +93,16 @@ class HackingTestCase(test.TestCase): self.assertEqual(len(list(checks.no_translate_debug_logs( "LOG.info(_('foo'))"))), 0) + def test_no_use_conf_debug_check(self): + self.assertEqual(len(list(checks.no_use_conf_debug_check( + "if CONF.debug:", "fakefile"))), 1) + + self.assertEqual(len(list(checks.no_use_conf_debug_check( + "if cfg.CONF.debug", "fakefile"))), 1) + + self.assertEqual(len(list(checks.no_use_conf_debug_check( + "if logging.is_debug()", "fakefile"))), 0) + def test_assert_true_instance(self): self.assertEqual(len(list(checks.assert_true_instance( "self.assertTrue(isinstance(e, "