diff --git a/README.rst b/README.rst index 1141bf2..1c05fd5 100644 --- a/README.rst +++ b/README.rst @@ -5,13 +5,13 @@ refstack-client is a command line utility that allows you to execute Tempest test runs based on configurations you specify. When finished running Tempest it sends the passed test data back to the Refstack API server. -**Usage** +**Setup** We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe. -$ ./setup_env.sh +$ ./setup_env -**Start testing** +**Usage** 1. Prepare a tempest configuration file that is customized to your cloud environment. @@ -21,10 +21,9 @@ $ ./setup_env.sh source .venv/bin/activate -4. Execute test by typing: +4. Test your cloud by typing: - ./refstack-client -c "Path of the tempest configuration file\ - to use" + ./refstack-client test -c **Note:** @@ -33,5 +32,21 @@ $ ./setup_env.sh c. Adding -t option will only test a particular test case or a test group. This option can be used for quick verification of the target test cases (i.e. -t "tempest.api.identity.admin.test_roles"). - d. Adding --url option will upload the test results to a Refstack API server - instead of the default Refstack API server. + d. Adding --url option will upload the test results to the specified + Refstack API server instead of the default Refstack API server. + server instead of the default Refstack API server. + e. Adding --offline option will have your test results not be uploaded. + + + **Upload:** + + If you previously ran a test with refstack-client using the --offline + option, you can upload your results to a Refstack API server by using the + following command: + + ./refstack-client upload + + The results file is a JSON file generated by refstack-client when a test has + completed. This is saved in .venv/src/tempest/.testrepository. When you use + the 'upload' command, you can also override the Refstack API server + uploaded to with the --url option. diff --git a/refstack-client b/refstack-client index 3978d9e..bc70273 100755 --- a/refstack-client +++ b/refstack-client @@ -28,4 +28,4 @@ from refstack_client import refstack_client if __name__ == '__main__': args = refstack_client.parse_cli_args() test = refstack_client.RefstackClient(args) - test.run() \ No newline at end of file + getattr(test, args.func)() \ No newline at end of file diff --git a/refstack_client/refstack_client.py b/refstack_client/refstack_client.py index afc93e4..c437e37 100755 --- a/refstack_client/refstack_client.py +++ b/refstack_client/refstack_client.py @@ -48,16 +48,17 @@ class RefstackClient: logging.Formatter(self.log_format)) self.logger.addHandler(self.console_log_handle) - if args.verbose > 1: + self.args = args + + if self.args.verbose > 1: self.logger.setLevel(logging.DEBUG) - elif args.verbose == 1: + elif self.args.verbose == 1: self.logger.setLevel(logging.INFO) else: self.logger.setLevel(logging.ERROR) - self.args = args - self.tempest_script = os.path.join(self.args.tempest_dir, - 'run_tempest.sh') + def _prep_test(self): + '''Prepare a tempest test against a cloud.''' # Check that the config file exists. if not os.path.isfile(self.args.conf_file): @@ -76,6 +77,16 @@ class RefstackClient: self.conf_file = self.args.conf_file self.conf = ConfigParser.SafeConfigParser() self.conf.read(self.args.conf_file) + self.tempest_script = os.path.join(self.args.tempest_dir, + 'run_tempest.sh') + + def _prep_upload(self): + '''Prepare an upload to the Refstack_api''' + if not os.path.isfile(self.args.file): + self.logger.error("File not valid: %s" % self.args.file) + exit(1) + self.logger.setLevel(logging.DEBUG) + self.upload_file = self.args.file def _get_next_stream_subunit_output_file(self, tempest_dir): '''This method reads from the next-stream file in the .testrepository @@ -131,6 +142,12 @@ class RefstackClient: content['results'] = results return content + def _save_json_results(self, results, path): + '''Save the output results from the Tempest run as a JSON file''' + file = open(path, "w+") + file.write(json.dumps(results, indent=4, separators=(',', ': '))) + file.close() + def get_passed_tests(self, result_file): '''Get a list of tests IDs that passed Tempest from a subunit file.''' subunit_processor = SubunitProcessor(result_file) @@ -145,8 +162,9 @@ class RefstackClient: json_content = json.dumps(content) self.logger.debug('API request content: %s ' % json_content) - def run(self): - '''Execute tempest test against the cloud.''' + def test(self): + '''Execute Tempest test against the cloud.''' + self._prep_test() results_file = self._get_next_stream_subunit_output_file( self.args.tempest_dir) cpid = self._get_cpid_from_keystone(self.conf) @@ -189,6 +207,11 @@ class RefstackClient: results = self.get_passed_tests(results_file) self.logger.info("Number of passed tests: %d" % len(results)) + content = self._form_result_content(cpid, duration, results) + json_path = results_file + ".json" + self._save_json_results(content, json_path) + self.logger.info('JSON results saved in: %s' % json_path) + # If the user did not specify the offline argument, then upload # the results. if not self.args.offline: @@ -198,50 +221,82 @@ class RefstackClient: self.logger.error("Problem executing Tempest script. Exit code %d", process.returncode) + def upload(self): + '''Perform upload to Refstack URL.''' + self._prep_upload() + json_file = open(self.upload_file) + json_data = json.load(json_file) + json_file.close() + self.post_results(self.args.url, json_data) + def parse_cli_args(args=None): - parser = argparse.ArgumentParser(description='Starts a tempest test', + usage_string = ('refstack-client [-h] {upload,test} ...\n\n' + 'To see help on specific argument, do:\n' + 'refstack-client -h') + + parser = argparse.ArgumentParser(description='Refstack-client arguments', formatter_class=argparse. - ArgumentDefaultsHelpFormatter) + ArgumentDefaultsHelpFormatter, + usage=usage_string) - parser.add_argument('-v', '--verbose', - action='count', - help='Show verbose output. Note that -vv will show ' - 'Tempest test result output.') + subparsers = parser.add_subparsers(help='Available subcommands.') - parser.add_argument('--offline', - action='store_true', - help='Do not upload test results after running ' - 'Tempest.') + # Arguments that go with all subcommands. + shared_args = argparse.ArgumentParser(add_help=False) + shared_args.add_argument('-v', '--verbose', + action='count', + help='Show verbose output.') - parser.add_argument('--url', - action='store', - required=False, - default='https://api.refstack.org', - type=str, - help='Refstack API URL to post results to (e.g. --url ' - 'https://127.0.0.1:8000).') + shared_args.add_argument('--url', + action='store', + required=False, + default='https://api.refstack.org', + type=str, + help='Refstack API URL to upload results to ' + '(--url https://127.0.0.1:8000).') - parser.add_argument('--tempest-dir', - action='store', - required=False, - dest='tempest_dir', - default='.venv/src/tempest', - help='Path of the tempest project directory.') + # Upload command + parser_upload = subparsers.add_parser('upload', parents=[shared_args], + help='Upload an existing result ' + 'file. ') + parser_upload.add_argument('file', + type=str, + help='Path of JSON results file.') + parser_upload.set_defaults(func="upload") - parser.add_argument('-c', '--conf-file', - action='store', - required=True, - dest='conf_file', - type=str, - help='Path of the tempest configuration file to use.') + # Test command + parser_test = subparsers.add_parser('test', parents=[shared_args], + help='Run Tempest against a cloud.') + + parser_test.add_argument('--tempest-dir', + action='store', + required=False, + dest='tempest_dir', + default='.venv/src/tempest', + help='Path of the Tempest project directory.') + + parser_test.add_argument('-c', '--conf-file', + action='store', + required=True, + dest='conf_file', + type=str, + help='Path of the Tempest configuration file to ' + 'use.') + + parser_test.add_argument('-t', '--test-cases', + action='store', + required=False, + dest='test_cases', + type=str, + help='Specify a subset of test cases to run ' + '(e.g. --test-cases tempest.api.compute).') + + parser_test.add_argument('--offline', + action='store_true', + help='Do not upload test results after running ' + 'Tempest.') + parser_test.set_defaults(func="test") - parser.add_argument('-t', '--test-cases', - action='store', - required=False, - dest='test_cases', - type=str, - help='Specify a subset of test cases to run ' - '(e.g. --test-cases tempest.api.compute).') return parser.parse_args(args=args) diff --git a/refstack_client/tests/unit/.testrepository/0.json b/refstack_client/tests/unit/.testrepository/0.json new file mode 100644 index 0000000..5b24fb4 --- /dev/null +++ b/refstack_client/tests/unit/.testrepository/0.json @@ -0,0 +1,8 @@ +{ + "duration_seconds": 0, + "cpid": "test-id", + "results": [ + "tempest.passed.test" + ] +} + diff --git a/refstack_client/tests/unit/test_client.py b/refstack_client/tests/unit/test_client.py index 2645188..2a6db8e 100755 --- a/refstack_client/tests/unit/test_client.py +++ b/refstack_client/tests/unit/test_client.py @@ -15,7 +15,9 @@ # import logging +import json import os +import tempfile import mock from mock import MagicMock @@ -51,7 +53,8 @@ class TestRefstackClient(unittest.TestCase): conf_file_name = self.conf_file_name if tempest_dir is None: tempest_dir = self.test_path - argv = ['-c', conf_file_name, + argv = ['test', + '-c', conf_file_name, '--tempest-dir', tempest_dir, '--test-cases', 'tempest.api.compute', '--url', '0.0.0.0'] @@ -86,14 +89,17 @@ class TestRefstackClient(unittest.TestCase): """ args = rc.parse_cli_args(self.mock_argv()) client = rc.RefstackClient(args) + client._prep_test() self.assertEqual(client.logger.level, logging.ERROR) args = rc.parse_cli_args(self.mock_argv(verbose='-v')) client = rc.RefstackClient(args) + client._prep_test() self.assertEqual(client.logger.level, logging.INFO) args = rc.parse_cli_args(self.mock_argv(verbose='-vv')) client = rc.RefstackClient(args) + client._prep_test() self.assertEqual(client.logger.level, logging.DEBUG) def test_get_next_stream_subunit_output_file(self): @@ -128,6 +134,7 @@ class TestRefstackClient(unittest.TestCase): """ args = rc.parse_cli_args(self.mock_argv()) client = rc.RefstackClient(args) + client._prep_test() self.mock_keystone() cpid = client._get_cpid_from_keystone(client.conf) self.ks_client_builder.assert_called_with( @@ -142,6 +149,7 @@ class TestRefstackClient(unittest.TestCase): """ args = rc.parse_cli_args(self.mock_argv()) client = rc.RefstackClient(args) + client._prep_test() client.conf.remove_option('identity', 'admin_tenant_id') client.conf.set('identity', 'admin_tenant_name', 'admin_tenant_name') self.mock_keystone() @@ -159,6 +167,7 @@ class TestRefstackClient(unittest.TestCase): """ args = rc.parse_cli_args(self.mock_argv(verbose='-vv')) client = rc.RefstackClient(args) + client._prep_test() client.conf.remove_option('identity', 'admin_tenant_id') self.assertRaises(SystemExit, client._get_cpid_from_keystone, client.conf) @@ -175,6 +184,25 @@ class TestRefstackClient(unittest.TestCase): 'results': ['tempest.sample.test']} self.assertEqual(expected, content) + def test_save_json_result(self): + """ + Test that the results are properly written to a JSON file. + """ + args = rc.parse_cli_args(self.mock_argv()) + client = rc.RefstackClient(args) + results = {'cpid': 1, + 'duration_seconds': 1, + 'results': ['tempest.sample.test']} + temp_file = tempfile.NamedTemporaryFile() + client._save_json_results(results, temp_file.name) + + # Get the JSON that was written to the file and make sure it + # matches the expected value. + json_file = open(temp_file.name) + json_data = json.load(json_file) + json_file.close() + self.assertEqual(results, json_data) + def test_get_passed_tests(self): """ Test that only passing tests are retrieved from a subunit file. @@ -201,7 +229,7 @@ class TestRefstackClient(unittest.TestCase): client.get_passed_tests = MagicMock(return_value=['test']) client.post_results = MagicMock() client._save_json_results = MagicMock() - client.run() + client.test() mock_popen.assert_called_with( ('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, '-N', '-t', '--', 'tempest.api.compute'), @@ -229,7 +257,7 @@ class TestRefstackClient(unittest.TestCase): client.get_passed_tests = MagicMock(return_value=['test']) client.post_results = MagicMock() client._save_json_results = MagicMock() - client.run() + client.test() mock_popen.assert_called_with( ('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, '-N', '-t', '--', 'tempest.api.compute'), @@ -245,14 +273,16 @@ class TestRefstackClient(unittest.TestCase): Test when a nonexistent configuration file is passed in. """ args = rc.parse_cli_args(self.mock_argv(conf_file_name='ptn-khl')) - self.assertRaises(SystemExit, rc.RefstackClient, args) + client = rc.RefstackClient(args) + self.assertRaises(SystemExit, client.test) def test_run_tempest_nonexisting_directory(self): """ Test when a nonexistent Tempest directory is passed in. """ args = rc.parse_cli_args(self.mock_argv(tempest_dir='/does/not/exist')) - self.assertRaises(SystemExit, rc.RefstackClient, args) + client = rc.RefstackClient(args) + self.assertRaises(SystemExit, client.test) def test_failed_run(self): """ @@ -264,5 +294,33 @@ class TestRefstackClient(unittest.TestCase): args = rc.parse_cli_args(self.mock_argv(verbose='-vv')) client = rc.RefstackClient(args) client.logger.error = MagicMock() - client.run() + client.test() self.assertTrue(client.logger.error.called) + + def test_upload(self): + """ + Test that the upload command runs as expected. + """ + upload_file_path = self.test_path + "/.testrepository/0.json" + args = rc.parse_cli_args(['upload', upload_file_path, + '--url', 'http://api.test.org']) + client = rc.RefstackClient(args) + + client.post_results = MagicMock() + client.upload() + expected_json = {'duration_seconds': 0, + 'cpid': 'test-id', + 'results': ['tempest.passed.test']} + + client.post_results.assert_called_with('http://api.test.org', + expected_json) + + def test_upload_nonexisting_file(self): + """ + Test that the upload file does not exist + """ + upload_file_path = self.test_path + "/.testrepository/foo.json" + args = rc.parse_cli_args(['upload', upload_file_path, + '--url', 'http://api.test.org']) + client = rc.RefstackClient(args) + self.assertRaises(SystemExit, client.upload)