191 lines
6.3 KiB
Python
191 lines
6.3 KiB
Python
# coding=utf-8
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import argparse
|
|
import functools
|
|
import json
|
|
import logging
|
|
import math
|
|
import os
|
|
|
|
import jinja2
|
|
from tabulate import tabulate
|
|
import yaml
|
|
|
|
from rally_runners.reliability import analytics
|
|
from rally_runners.reliability import graphics
|
|
from rally_runners import utils
|
|
|
|
REPORT_TEMPLATE = 'rally_runners/reliability/templates/report.rst'
|
|
SCENARIOS_DIR = 'rally_runners/reliability/scenarios/'
|
|
|
|
|
|
def round2(number, variance=None):
|
|
if not variance:
|
|
variance = number
|
|
return round(number, int(math.ceil(-(math.log10(variance)))) + 1)
|
|
|
|
|
|
def mean_var_to_str(mv):
|
|
if not mv:
|
|
return 'N/A'
|
|
|
|
if mv.var == 0:
|
|
precision = 4
|
|
else:
|
|
precision = int(math.ceil(-(math.log10(mv.var)))) + 1
|
|
if precision > 0:
|
|
pattern = '%%.%df' % precision
|
|
pattern_1 = '%%.%df' % (precision)
|
|
else:
|
|
pattern = pattern_1 = '%d'
|
|
|
|
return '%s ~%s' % (pattern % round(mv.statistic, precision),
|
|
pattern_1 % round(mv.var, precision + 1))
|
|
|
|
|
|
def tabulate2(*args, **kwargs):
|
|
return (u'%s' % tabulate(*args, **kwargs)).replace(' ~', u'\u00A0±')
|
|
|
|
|
|
def get_runs(raw_rally_reports):
|
|
for one_report in raw_rally_reports:
|
|
for one_run in one_report:
|
|
yield one_run
|
|
|
|
|
|
def indent(text, distance):
|
|
return '\n'.join((' ' * distance + line) for line in text.split('\n'))
|
|
|
|
|
|
def process(raw_rally_reports, book_folder, scenario, scenario_name):
|
|
scenario_text = indent(scenario, 4)
|
|
report = dict(runs=[], scenario=scenario_text, scenario_name=scenario_name)
|
|
|
|
summary = analytics.process_all_runs(get_runs(raw_rally_reports))
|
|
logging.debug('Summary: %s', summary)
|
|
|
|
has_errors = False
|
|
has_degradation = False
|
|
|
|
for i, one_run in enumerate(summary.run_results):
|
|
report_one_run = {}
|
|
|
|
plot = graphics.draw_plot(one_run)
|
|
plot.savefig(os.path.join(book_folder, 'plot_%d.svg' % (i + 1)))
|
|
|
|
headers = ['Samples', 'Median, s', 'Mean, s', 'Std dev',
|
|
'95% percentile, s']
|
|
t = [[one_run.etalon_stats.count,
|
|
round2(one_run.etalon_stats.median),
|
|
round2(one_run.etalon_stats.mean),
|
|
round2(one_run.etalon_stats.std),
|
|
round2(one_run.etalon_stats.p95)]]
|
|
report_one_run['etalon_table'] = tabulate2(
|
|
t, headers=headers, tablefmt='grid')
|
|
|
|
headers = ['#', 'Downtime, s']
|
|
t = []
|
|
for index, stat in enumerate(one_run.error_area):
|
|
t.append([index + 1, mean_var_to_str(stat.duration)])
|
|
|
|
if one_run.error_area:
|
|
has_errors = True
|
|
report_one_run['errors_table'] = tabulate2(
|
|
t, headers=headers, tablefmt='grid')
|
|
|
|
headers = ['#', 'Time to recover, s', 'Absolute degradation, s',
|
|
'Relative degradation']
|
|
t = []
|
|
for index, stat in enumerate(one_run.degradation_area):
|
|
t.append([index + 1,
|
|
mean_var_to_str(stat.duration),
|
|
mean_var_to_str(stat.degradation),
|
|
mean_var_to_str(stat.degradation_ratio)])
|
|
|
|
if one_run.degradation_area:
|
|
has_degradation = True
|
|
report_one_run['degradation_table'] = tabulate2(
|
|
t, headers=headers, tablefmt="grid")
|
|
|
|
report['runs'].append(report_one_run)
|
|
|
|
headers = ['Service downtime, s', 'MTTR, s',
|
|
'Absolute performance degradation, s',
|
|
'Relative performance degradation, ratio']
|
|
t = [[mean_var_to_str(summary.downtime),
|
|
mean_var_to_str(summary.mttr),
|
|
mean_var_to_str(summary.degradation),
|
|
mean_var_to_str(summary.degradation_ratio)]]
|
|
report['summary_table'] = tabulate2(t, headers=headers, tablefmt='grid')
|
|
|
|
report['has_errors'] = has_errors
|
|
report['has_degradation'] = has_degradation
|
|
|
|
jinja_env = jinja2.Environment()
|
|
jinja_env.filters['json'] = json.dumps
|
|
jinja_env.filters['yaml'] = functools.partial(
|
|
yaml.safe_dump, indent=2, default_flow_style=False)
|
|
|
|
path = utils.resolve_relative_path(REPORT_TEMPLATE)
|
|
with open(path) as fd:
|
|
template = fd.read()
|
|
compiled_template = jinja_env.from_string(template)
|
|
rendered_template = compiled_template.render(dict(report=report))
|
|
|
|
index_path = os.path.join(book_folder, 'index.rst')
|
|
with open(index_path, 'w') as fd2:
|
|
fd2.write(rendered_template.encode('utf8'))
|
|
|
|
logging.info('The book is written to: %s', book_folder)
|
|
|
|
|
|
def make_report(scenario_name, raw_rally_file_names, book_folder):
|
|
scenario_dir = utils.resolve_relative_path(SCENARIOS_DIR)
|
|
scenario_path = os.path.join(scenario_dir, scenario_name)
|
|
if not scenario_path.endswith('.yaml'):
|
|
scenario_path += '.yaml'
|
|
|
|
with open(scenario_path) as fd:
|
|
scenario = fd.read()
|
|
|
|
raw_rally_reports = []
|
|
for file_name in raw_rally_file_names:
|
|
with open(file_name) as fd:
|
|
raw_rally_reports.append(json.loads(fd.read()))
|
|
|
|
utils.mkdir_tree(book_folder)
|
|
process(raw_rally_reports, book_folder, scenario, scenario_name)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(prog='rally-reliability-report')
|
|
parser.add_argument('-d', '--debug', action='store_true')
|
|
parser.add_argument('-i', '--input', dest='input', nargs='+',
|
|
help='Rally raw json output')
|
|
parser.add_argument('-b', '--book', dest='book', required=True,
|
|
help='folder where to write RST book')
|
|
parser.add_argument('-s', '--scenario', dest='scenario', required=True,
|
|
help='Rally scenario')
|
|
args = parser.parse_args()
|
|
|
|
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
|
|
level=logging.DEBUG if args.debug else logging.INFO)
|
|
|
|
make_report(args.scenario, args.input, args.book)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|