Enhance Rally HTML reports

This patch introduces the following changes.
1. We often have multiple executions of the same atomic action in
a single rally iteration. Existing rally charts do not show duration
for duplicate actions in an iteration accurately. This patch introduces
line charts for each occurence of a duplicate atomic action for each
instance, by passing it as additive data.
2. This patch also adds a per iteration stacked area chart that shows
data per iteration for each occurence of all atomic actions in the iteration.
3. This patch also adds a duration line chart for each resource created by
Rally.

Co-authored-by: venkata anil <anilvenkata@redhat.com>
Change-Id: I44dafad69cdbcd6db7c8fd9148f1b35d89924e03

browbeat-config.yaml

@@ -57,6 +57,7 @@ rally:
     - browbeat: rally/rally-plugins/browbeat
     - workloads: rally/rally-plugins/workloads
     - dynamic-workloads: rally/rally-plugins/dynamic-workloads
+    - reports: rally/rally-plugins/reports
 shaker:
   server: 1.1.1.1
   port: 5555

browbeat/workloads/rally.py

@@ -55,12 +55,12 @@ class Rally(base.WorkloadBase):
                 for plugin in self.config['rally']['plugins']:
                     for name in plugin:
                         plugins.append(plugin[name])
-        plugin_string = ""
+        self.plugin_string = ""
         if len(plugins) > 0:
-            plugin_string = "--plugin-paths {}".format(",".join(plugins))
+            self.plugin_string = "--plugin-paths {}".format(",".join(plugins))
         cmd = "source {}; ".format(get_workload_venv('rally', True))
         cmd += "rally {} task start {} --task-args \'{}\' 2>&1 | tee {}.log".format(
-            plugin_string, task_file, task_args, test_name)
+            self.plugin_string, task_file, task_args, test_name)
         from_time = time.time()
         self.tools.run_cmd(cmd)['stdout']
         to_time = time.time()
@@ -79,18 +79,19 @@ class Rally(base.WorkloadBase):
     def gen_scenario_html(self, task_ids, test_name):
         all_task_ids = ' '.join(task_ids)
         cmd = "source {}; ".format(get_workload_venv('rally', True))
-        cmd += "rally task report --uuid {} --out {}.html".format(
+        cmd += "rally {} task report --uuid {} --out {}.html".format(
-            all_task_ids, test_name)
+            self.plugin_string, all_task_ids, test_name)
         return self.tools.run_cmd(cmd)['stdout']
 
     def gen_scenario_json(self, task_id):
         cmd = "source {}; ".format(get_workload_venv('rally', True))
-        cmd += "rally task results --uuid {}".format(task_id)
+        cmd += "rally {} task results --uuid {}".format(self.plugin_string, task_id)
         return self.tools.run_cmd(cmd)['stdout']
 
     def gen_scenario_json_file(self, task_id, test_name):
         cmd = "source {}; ".format(get_workload_venv('rally', True))
-        cmd += "rally task results --uuid {} > {}.json".format(task_id, test_name)
+        cmd += "rally {} task results --uuid {} > {}.json".format(self.plugin_string,
+                                                                  task_id, test_name)
         return self.tools.run_cmd(cmd)['stdout']
 
     def rally_metadata(self, result, meta):

doc/source/plugins.rst

@@ -44,3 +44,77 @@ Scenario - nova_boot_persist_with_network_volume_fip
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 This scenario creates instances with a nic, a volume and associates a floating ip that persist upon completion of a rally run.  It is used as a workload with Telemetry by spawning many instances that have many metrics for the Telemetry subsystem to collect upon.
+
+Charts
+^^^^^^
+
+To include any of the custom charts from Browbeat in a scenario, the following lines will have to be included in the python file of the program.
+.. code-block:: python
+
+   import sys
+   import os
+
+   sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../reports')))
+   from generate_scenario_duration_charts import ScenarioDurationChartsGenerator  # noqa: E402
+
+The customc charts will appear in the "Scenario Data" section of the Rally HTML report.
+
+Chart - add_per_iteration_complete_data
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This plugin generates a stacked area graph for duration trend for each atomic action in an iteration.
+To include this chart in any scenario, add the following lines at the end of the run() function of the scenario in the python file.
+
+.. code-block:: python
+
+   self.duration_charts_generator = ScenarioDurationChartsGenerator()
+   self.duration_charts_generator.add_per_iteration_complete_data(self)
+
+The graphs will appear under the "Per iteration" section of "Scenario Data" in the Rally HTML report.
+The resulting graphs will look like the images below.
+
+.. image:: images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration1.png
+   :alt: Iteration 1 Chart
+
+.. image:: images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration2.png
+   :alt: Iteration 2 Chart
+
+.. image:: images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration3.png
+   :alt: Iteration 3 Chart
+
+.. image:: images/Per_Iteration_Duration_Stacked_Area_Chart/Iteration4.png
+   :alt: Iteration 4 Chart
+
+Chart - add_duplicate_atomic_actions_iteration_additive_data
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This plugin generates line graphs for atomic actions that have been executed more than once in the same iteration.
+To include this chart in any scenario, add the following lines at the end of the run() function of the scenario in the python file.
+
+.. code-block:: python
+
+   self.duration_charts_generator = ScenarioDurationChartsGenerator()
+   self.duration_charts_generator.add_duplicate_atomic_actions_iteration_additive_data(self)
+
+The graphs will	appear under the "Aggregated" section of "Scenario Data" in the Rally HTML report.
+The resulting graphs will look like the	images below.
+
+.. image:: images/Duplicate_Atomic_Actions_Duration_Line_Chart.png
+   :alt: Duplicate Atomic Actions Duration Line Chart
+
+Chart - add_all_resources_additive_data
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This plugin generates a line graph for duration data from each resource created by Rally.
+To include this chart in any scenario, add the following lines at the end of the run() function of the scenario in the python file.
+
+.. code-block:: python
+
+   self.duration_charts_generator = ScenarioDurationChartsGenerator()
+   self.duration_charts_generator.add_all_resources_additive_data(self)
+
+The graphs will appear under the "Aggregated" section of "Scenario Data" in the Rally HTML report.
+The resulting graphs will look like the images below.
+
+.. image:: images/Resource_Atomic_Actions_Duration_Line_Chart.png
+   :alt: Resource Atomic Actions Duration Line Chart

rally/rally-plugins/netcreate-boot/netcreate_nova_boot_vms_on_single_network.py

@@ -17,6 +17,11 @@ from rally.task import scenario
 from rally.task import types
 from rally.task import validation
 
+import sys
+import os
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../reports')))
+from generate_scenario_duration_charts import ScenarioDurationChartsGenerator  # noqa: E402
 
 @types.convert(image={"type": "glance_image"}, flavor={"type": "nova_flavor"})
 @validation.add("image_valid_on_flavor", flavor_param="flavor", image_param="image")
@@ -38,3 +43,8 @@ class CreateVMsOnSingleNetwork(neutron_utils.NeutronScenario,
             port = self._create_port(network, port_create_args or {})
             kwargs["nics"].append({'port-id': port['port']['id']})
             self._boot_server(image, flavor, **kwargs)
+
+        self.duration_charts_generator = ScenarioDurationChartsGenerator()
+        self.duration_charts_generator.add_per_iteration_complete_data(self)
+        self.duration_charts_generator.add_duplicate_atomic_actions_iteration_additive_data(self)
+        self.duration_charts_generator.add_all_resources_additive_data(self)

rally/rally-plugins/reports/custom_chart_plugins.py

@@ -0,0 +1,81 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+from rally.common.plugin import plugin
+from rally.task.processing import charts
+from rally.task.processing import utils
+
+@plugin.configure(name="ResourceDurationLines")
+class ResourceDurationOutputLinesChart(charts.OutputStackedAreaChart):
+    """Display resource duration data as generic chart with lines.
+
+    This plugin processes additive data and displays it in HTML report
+    as linear chart with X axis bound to iteration number.
+    Complete output data is displayed as linear chart as well, without
+    any processing.
+
+    Examples of using this plugin in Scenario, for saving output data:
+
+    .. code-block:: python
+
+        self.add_output(
+            additive={"title": "Resources atomic action duration line chart",
+                      "description": "Resources trend",
+                      "chart_plugin": "ResourceDurationLines",
+                      "data": [["foo", 12], ["bar", 34]],
+                      "label": "Duration(in seconds)",
+                      "axis_label": "Resource count"})
+    """
+
+    widget = "Lines"
+
+    def add_iteration(self, iteration):
+        """Add iteration data.
+        This method must be called for each iteration.
+        :param iteration: list, resource duration data for current iteration
+        """
+        atomic_count = {}
+        self.max_count = 0
+        for name, value in iteration:
+            if name not in atomic_count.keys():
+                atomic_count[name] = 1
+            else:
+                atomic_count[name] += 1
+            self.max_count = max(self.max_count, atomic_count[name])
+
+        for name, value in iteration:
+            if name not in self._data:
+                self._data[name] = utils.GraphZipper(self.base_size*self.max_count,
+                                                     self.zipped_size)
+            self._data[name].add_point(value)
+
+    def zeropad_duration_data(self):
+        """Some actions might have been executed more times than other actions
+        in the same iteration. Zeroes are appended to make the numbers equal.
+        """
+        for key in self._data:
+            i = len(self._data[key].get_zipped_graph())
+            while i < self.base_size*self.max_count:
+                i += 1
+                self._data[key].add_point(0)
+
+    def render(self):
+        """Render HTML from resource duration data"""
+        self.zeropad_duration_data()
+        render_data = [(name, points.get_zipped_graph())
+                       for name, points in self._data.items()]
+        return {"title": self.title,
+                "description": self.description,
+                "widget": self.widget,
+                "data": render_data,
+                "label": self.label,
+                "axis_label": self.axis_label}

rally/rally-plugins/reports/generate_scenario_duration_charts.py

@@ -0,0 +1,69 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+from process_atomic_actions_data import AtomicActionsDurationDataProcessor
+
+class ScenarioDurationChartsGenerator(AtomicActionsDurationDataProcessor):
+    """Send processed complete and additive atomic action duration data to Rally"""
+
+    def add_per_iteration_complete_data(self, scenario_object):
+        """Generate a stacked area graph for duration trend for each atomic action
+        in an iteration.
+        :param scenario_object: Rally scenario object
+        """
+        atomic_actions = scenario_object.atomic_actions()
+        self.process_atomic_actions_complete_data(atomic_actions)
+        # Some actions might have been executed more times than other actions
+        # in the same iteration. Zeroes are appended to make the numbers equal.
+        self.zeropad_duration_data()
+        scenario_object.add_output(complete={
+                                   "title": "Atomic actions duration data as stacked area",
+                                   "description": "Iterations trend",
+                                   "chart_plugin": "StackedArea",
+                                   "data": (
+                                       self.get_complete_duration_data()),
+                                   "label": "Duration(in seconds)",
+                                   "axis_label": "Atomic action"})
+
+    def add_duplicate_atomic_actions_iteration_additive_data(self, scenario_object):
+        """Generate line graphs for atomic actions that have been executed more than once
+        in the same iteration.
+        :param scenario_object: Rally scenario object
+        """
+        atomic_actions = scenario_object.atomic_actions()
+        for action_name in self.get_duplicate_actions_list(atomic_actions):
+            additive_data_duplicate_action = (
+                self.process_atomic_action_additive_data(action_name,
+                                                         atomic_actions))
+            scenario_object.add_output(additive={
+                                       "title": "{} additive duration data as line chart".format(
+                                                action_name),
+                                       "description": "Iterations trend",
+                                       "chart_plugin": "Lines",
+                                       "data": additive_data_duplicate_action,
+                                       "label": "Duration(in seconds)"})
+
+    def add_all_resources_additive_data(self, scenario_object):
+        """Generate a line graph for duration data from each resource created by Rally.
+        :param scenario_object: Rally scenario object
+        """
+        atomic_actions = scenario_object.atomic_actions()
+        additive_data_all_actions = []
+        for action in atomic_actions:
+            additive_data_all_actions.append([action["name"], action["finished_at"] -
+                                              action["started_at"]])
+        scenario_object.add_output(additive={"title": "Resources atomic action duration line chart",
+                                             "description": "Resources trend",
+                                             "chart_plugin": "ResourceDurationLines",
+                                             "data": additive_data_all_actions,
+                                             "label": "Duration(in seconds)",
+                                             "axis_label": "Resource count"})

rally/rally-plugins/reports/process_atomic_actions_data.py

@@ -0,0 +1,77 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+class AtomicActionsDurationDataProcessor:
+    """Generate charts for atomic actions durations additive and complete data"""
+
+    def __init__(self):
+        self._duration_data = {}
+        self.max_num_data_points = 0
+
+    def process_atomic_actions_complete_data(self, atomic_actions):
+        """Generate duration data in complete format for per iteration chart
+        :param atomic_actions: list, self.atomic_actions() from a rally scenario
+        """
+        for action in atomic_actions:
+            if action["name"] not in self._duration_data:
+                self._duration_data[action["name"]] = []
+            action_duration = action["finished_at"] - action["started_at"]
+            self._duration_data[action["name"]].append([len(self._duration_data[action["name"]]),
+                                                       action_duration])
+            self.max_num_data_points = max(self.max_num_data_points,
+                                           len(self._duration_data[action["name"]]))
+
+    def get_duplicate_actions_list(self, atomic_actions):
+        """Get list of atomic actions which occur more than once in an iteration
+        :param atomic_actions: list, self.atomic_actions() from a rally scenario
+        :returns: list of strings representing duplicate action names
+        """
+        actions_set = set()
+        duplicate_actions_list = []
+        for action in atomic_actions:
+            if action["name"] not in actions_set:
+                actions_set.add(action["name"])
+            else:
+                duplicate_actions_list.append(action["name"])
+        return duplicate_actions_list
+
+    def process_atomic_action_additive_data(self, action_name, atomic_actions):
+        """Generate duration data in additive format for aggregate chart
+        :param action_name: str, action name to generate duration data for
+        :param atomic_actions: list, self.atomic_actions() from a rally scenario
+        :returns: list in Rally additive data format
+        """
+        additive_duration_data = []
+        action_index = 1
+        for action in atomic_actions:
+            if action["name"] == action_name:
+                additive_duration_data.append(["{}({})".format(action_name, action_index),
+                                               action["finished_at"] - action["started_at"]])
+                action_index += 1
+        return additive_duration_data
+
+    def zeropad_duration_data(self):
+        """Some atomic actions occur more times than other atomic actions
+        within the same iteration. Zeroes are appended to make the length
+        of all atomic action lists the same
+        """
+        for action_name in self._duration_data:
+            while len(self._duration_data[action_name]) < self.max_num_data_points:
+                self._duration_data[action_name].append([len(self._duration_data[action_name]), 0])
+
+    def get_complete_duration_data(self):
+        """Complete duration data is stored in dict format to increase efficiency
+        of operations. Rally add_output() function expects a list as input. This
+        function converts the complete duration data to the format expected by the
+        Rally add_output() function.
+        """
+        return [[name, durations] for name, durations in self._duration_data.items()]