diff --git a/releasenotes/notes/add-graph-time-for-test-run-time-variance-6e61e0d9b4bb0cf9.yaml b/releasenotes/notes/add-graph-time-for-test-run-time-variance-6e61e0d9b4bb0cf9.yaml new file mode 100644 index 0000000..fbd1e69 --- /dev/null +++ b/releasenotes/notes/add-graph-time-for-test-run-time-variance-6e61e0d9b4bb0cf9.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + A new subunit2sql-graph type, `test_run_time`, is added. This will graph the + run time variance for the specified tests as box and wiskers plots. + - | + A new DB API function, get_run_times_all_test_runs(), was added. This will + return a all the individual durations for sucessful executions for tests. diff --git a/subunit2sql/analysis/graph.py b/subunit2sql/analysis/graph.py index 8c5403e..40a8eca 100644 --- a/subunit2sql/analysis/graph.py +++ b/subunit2sql/analysis/graph.py @@ -24,6 +24,7 @@ import subunit2sql.analysis.failures import subunit2sql.analysis.run_failure_rate import subunit2sql.analysis.run_time import subunit2sql.analysis.run_time_meta +import subunit2sql.analysis.test_run_time from subunit2sql import shell CONF = cfg.CONF @@ -57,7 +58,7 @@ def add_command_parsers(subparsers): graph_commands = {} # Put commands from in-tree commands on init list for command in ['failures', 'run_time', 'agg_count', 'dailycount', - 'run_failure_rate', 'run_time_meta']: + 'run_failure_rate', 'run_time_meta', 'test_run_time']: graph_commands[command] = getattr(subunit2sql.analysis, command) # Load any installed out of tree commands on the init list diff --git a/subunit2sql/analysis/test_run_time.py b/subunit2sql/analysis/test_run_time.py new file mode 100644 index 0000000..c5373fe --- /dev/null +++ b/subunit2sql/analysis/test_run_time.py @@ -0,0 +1,66 @@ +# Copyright 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime + +import matplotlib +import matplotlib.pyplot as plt +from oslo_config import cfg +import pandas as pd + +from subunit2sql.db import api + +CONF = cfg.CONF +matplotlib.style.use('ggplot') + + +def set_cli_opts(parser): + parser.add_argument('test_ids', nargs='*', + help='Test ids to graph the run time for, if none ' + 'are specified all tests will be used') + + +def generate_series(): + session = api.get_session() + + test_ids = None + if CONF.command.test_ids: + test_ids = CONF.command.test_ids + if CONF.start_date: + start_date = datetime.datetime.strptime(CONF.start_date, '%Y-%m-%d') + else: + start_date = None + if CONF.stop_date: + stop_date = datetime.datetime.strptime(CONF.stop_date, '%Y-%m-%d') + else: + stop_date = None + run_times = api.get_run_times_all_test_runs(tests=test_ids, + start_date=start_date, + stop_date=stop_date, + session=session) + df = pd.DataFrame(dict( + [(k, pd.Series(v)) for k, v in run_times.iteritems()])) + if not CONF.title: + title = "Run aggregate run time grouped by metadata" + else: + title = CONF.title + # NOTE(mtreinish): Decrease label font size for the worst case where we + # have tons of groups + matplotlib.rcParams['xtick.labelsize'] = '3' + plt.figure() + plt.title(title) + df.plot(kind='box', rot=90) + plt.ylabel('Time (sec.)') + plt.tight_layout() + plt.savefig(CONF.output, dpi=900) diff --git a/subunit2sql/db/api.py b/subunit2sql/db/api.py index 4d00414..a4b581c 100644 --- a/subunit2sql/db/api.py +++ b/subunit2sql/db/api.py @@ -1220,6 +1220,60 @@ def get_ids_for_all_tests(session=None): return db_utils.model_query(models.Test, session).values(models.Test.id) +def get_run_times_all_test_runs(tests=None, start_date=None, stop_date=None, + session=None): + """Return the all the individual duration times for each test_run + + This function will return a dictionary where each key is a test_id and + the value is a list of all the durations for each run of that test + + :param list tests: the list of test_ids to get results for, if none is + specified all tests + :param str start_date: The date to use as the start date for results + :param str stop_date: The date to use as the cutoff date for results + :param session: optional session object if one isn't provided a new session + will be acquired for the duration of this operation + + :return run_times: all the durations for test_runs grouped by test_id + :rtype: dict + """ + session = session or get_session() + run_times_query = db_utils.model_query(models.TestRun, session).filter( + models.TestRun.status == 'success').join( + models.Test, models.TestRun.test_id == models.Test.id) + if tests: + run_times_query = run_times_query.filter( + models.Test.test_id.in_(tests)) + if start_date: + run_times_query = run_times_query.filter( + models.TestRun.start_time >= start_date) + if stop_date: + run_times_query = run_times_query.filter( + models.TestRun.start_time <= stop_date) + run_times = run_times_query.values(models.Test.test_id, + models.TestRun.start_time, + models.TestRun.start_time_microsecond, + models.TestRun.stop_time, + models.TestRun.stop_time_microsecond) + run_times_dict = {} + for run_time in run_times: + test_id = run_time[0] + if run_time[1]: + start_time = run_time[1] + start_time = start_time.replace(microsecond=run_time[2]) + else: + continue + if run_time[3]: + stop_time = run_time[3] + stop_time = stop_time.replace(microsecond=run_time[4]) + duration = read_subunit.get_duration(start_time, stop_time) + if test_id in run_times_dict: + run_times_dict[test_id].append(duration) + else: + run_times_dict[test_id] = [duration] + return run_times_dict + + def get_run_times_grouped_by_run_metadata_key(key, start_date=None, stop_date=None, session=None, match_key=None, diff --git a/subunit2sql/tests/db/test_api.py b/subunit2sql/tests/db/test_api.py index c6d7357..a4bf852 100644 --- a/subunit2sql/tests/db/test_api.py +++ b/subunit2sql/tests/db/test_api.py @@ -248,6 +248,45 @@ class TestDatabaseAPI(base.TestCase): id_value = api.get_id_from_test_id('fake_test') self.assertEqual(test_a.id, id_value) + def test_get_run_times_all_test_runs(self): + timestamp_a = datetime.datetime.utcnow() + timestamp_b = timestamp_a + datetime.timedelta(seconds=3) + run_a = api.create_run() + run_b = api.create_run() + test_a = api.create_test('test_a') + test_b = api.create_test('test_b') + api.create_test_run(test_a.id, run_a.id, 'success', timestamp_a, + timestamp_b) + api.create_test_run(test_a.id, run_b.id, 'success', timestamp_a, + timestamp_b) + api.create_test_run(test_b.id, run_b.id, 'success', timestamp_a, + timestamp_b) + res = api.get_run_times_all_test_runs() + expected_dict = { + 'test_a': [3, 3], + 'test_b': [3] + } + self.assertEqual(expected_dict, res) + + def test_get_run_times_all_test_runs_with_tests_filter(self): + timestamp_a = datetime.datetime.utcnow() + timestamp_b = timestamp_a + datetime.timedelta(seconds=3) + run_a = api.create_run() + run_b = api.create_run() + test_a = api.create_test('test_a') + test_b = api.create_test('test_b') + api.create_test_run(test_a.id, run_a.id, 'success', timestamp_a, + timestamp_b) + api.create_test_run(test_a.id, run_b.id, 'success', timestamp_a, + timestamp_b) + api.create_test_run(test_b.id, run_b.id, 'success', timestamp_a, + timestamp_b) + res = api.get_run_times_all_test_runs(tests=['test_a']) + expected_dict = { + 'test_a': [3, 3], + } + self.assertEqual(expected_dict, res) + def test_get_test_runs_by_run_id(self): run_b = api.create_run() run_a = api.create_run()