Multiple worksheets for Rally json Google Sheets

This patch introduces multiple worksheets within one Google sheet
for multiple atomic actions

Change-Id: Ia6542cd7646a87c53cd633bde070939729a7ef9b

doc/source/usage.rst

@@ -343,7 +343,7 @@ Generate CSV file/Google Sheets from Rally json file
 --------------------------------------------
 Rally generates a json file with data about atomic actions duration from each iteration. These atomic actions often occur multiple times within one iteration.
 Browbeat has a script which allows a user to generate a CSV file and also has an option to generate a Google Sheet about individual resource
-duration through the Rally json file. To use the script to upload the CSV file to Google Sheets, a Google Drive service account is required.
+duration through the Rally json file. To use the script to upload to Google Sheets, a Google Drive service account is required.
 The script sends an email to the email id of the user with the Google sheet if the --uploadgooglesheet option is enabled.
 
 To generate only a CSV file and not upload to Google Sheets :
@@ -351,11 +351,27 @@ To generate only a CSV file and not upload to Google Sheets :
 ::
 
     $ source .browbeat-venv/bin/activate && cd utils
-    $ python rally_google_sheet_gen.py -c <path to csv file to write locally> -j <path to rally json file> -a <atomic action(Eg.: nova.boot_server>
+    $ python rally_google_sheet_gen.py -c -f <path to directory to write csv file locally>
+      -j <path to rally json file>
+      -a <space separated list of atomic actions(Eg.: boot_server create_network)>
+
+To only upload to Google Sheets and not generate CSV files :
+
+::
+    $ source .browbeat-venv/bin/activate && cd utils
+    $ python rally_google_sheet_gen.py
+      -j <path to rally json file>
+      -a <space separated list of atomic actions(Eg.: boot_server create_network)> -g                                                           
+      -s <path to google service account json credentials file>
+      -e <email id of user> -n <name of google sheet to be created>
+
 
 To generate a CSV file and upload to Google Sheets :
 
 ::
     $ source .browbeat-venv/bin/activate && cd utils
-    $ python rally_google_sheet_gen.py -c <path to csv file to write locally> -j <path to rally json file> -a <atomic action(Eg.: nova.boot_server>
+    $ python rally_google_sheet_gen.py -c -f <path to directory to write csv file locally>
-      -g -s <path to google service account json credentials file> -e <email id of user> -n <name of google sheet to be created>
+      -j <path to rally json file>
+      -a <space separated list of atomic actions(Eg.: boot_server create_network)> -g                                                           
+      -s <path to google service account json credentials file>
+      -e <email id of user> -n <name of google sheet to be created>

requirements.txt

@@ -14,3 +14,5 @@ pykwalify>=1.8.0;python_version>='3'
 gspread
 oauth2client
 gspread_formatting
+pandas
+gspread_dataframe

utils/rally_google_sheet_gen.py

@@ -20,57 +20,78 @@
 # The script sends an email to the email id of the user with the Google sheet
 # if the --uploadgooglesheet option is enabled.
 
-# To generate only a CSV file and not upload to Google Sheets :
+# To generate only CSV files and not upload to Google Sheets :
 
 #    $ source .browbeat-venv/bin/activate && cd utils
-#    $ python rally_google_sheet_gen.py -c <path to csv file to write locally>
+#    $ python rally_google_sheet_gen.py -c -f <path to directory to write csv file locally>
-#      -j <path to rally json file> -a <atomic action(Eg.: nova.boot_server>
+#      -j <path to rally json file>
+#      -a <space separated list of atomic actions(Eg.: boot_server create_network)>
+
+# To only upload to Google Sheets and not generate CSV files :
+
+#    $ source .browbeat-venv/bin/activate && cd utils
+#    $ python rally_google_sheet_gen.py
+#      -j <path to rally json file>
+#      -a <space separated list of atomic actions(Eg.: boot_server create_network)> -g
+#      -s <path to google service account json credentials file>
+#      -e <email id of user> -n <name of google sheet to be created>
 
 # To generate a CSV file and upload to Google Sheets :
 
 #    $ source .browbeat-venv/bin/activate && cd utils
-#    $ python rally_google_sheet_gen.py -c <path to csv file to write locally>
+#    $ python rally_google_sheet_gen.py -c -f <path to directory to write csv file locally>
-#      -j <path to rally json file> -a <atomic action(Eg.: nova.boot_server> -g
+#      -j <path to rally json file>
+#      -a <space separated list of atomic actions(Eg.: boot_server create_network)> -g
 #      -s <path to google service account json credentials file>
 #      -e <email id of user> -n <name of google sheet to be created>
 
 import json
-import csv
+import pandas as pd
 import gspread
 from oauth2client.service_account import ServiceAccountCredentials
 from gspread_formatting import cellFormat
 from gspread_formatting import format_cell_range
 from gspread_formatting import set_column_width
+from gspread_dataframe import set_with_dataframe
 import argparse
 
-def convert_rally_json_to_csv(json_file_path, atomic_action, csv_file_path):
+def add_atomic_action_data_to_df(json_file_path, atomic_action, df_dict):
     """Convert rally json data to csv file containing atomic action data
     :param json_file_path: str, path to rally json file
     :param atomic_action: str, atomic action to generate duration data
-    :param csv_file_path: str, path to write csv file
+    :param df_dict: dict, dict of dataframes from different atomic actions
     """
     json_file = open(json_file_path)
     json_data = json.load(json_file)
 
-    with open(csv_file_path, 'w', newline='') as csvfile:
+    df = pd.DataFrame({"Resource Number": [], "Duration(in seconds)": []})
-        csvwriter = csv.writer(csvfile, delimiter=',',
-                               quotechar='|', quoting=csv.QUOTE_MINIMAL)
 
-        csvwriter.writerow(["Resource Number", "Duration(in seconds)"])
+    resource_number = 1
-        resource_number = 1
 
-        for iteration in json_data[0]["result"]:
+    for iteration in json_data[0]["result"]:
-            for atomic_action_json in iteration["atomic_actions"]:
+        for atomic_action_json in iteration["atomic_actions"]:
-                if atomic_action in atomic_action_json:
+            if atomic_action in atomic_action_json:
-                    atomic_action_duration = iteration["atomic_actions"][atomic_action_json]
+                atomic_action_duration = iteration["atomic_actions"][atomic_action_json]
-                    csvwriter.writerow([resource_number, atomic_action_duration])
+                df = df.append({"Resource Number": resource_number,
-                    resource_number += 1
+                                "Duration(in seconds)": atomic_action_duration},
+                               ignore_index=True)
+                resource_number += 1
 
-    print("CSV file {} generated successfully.".format(csv_file_path))
+    df_dict[atomic_action] = df
+    print("Pandas DF for atomic action {} generated successfully.".format(atomic_action))
 
-def push_to_gsheet(csv_file_path, google_svc_acc_key, email_id):
+def generate_csv_from_df(csv_files_path, df_dict):
+    """Generate csv files from pandas dataframe
+    :param csv_files_path: str, path of directory to generate csv files
+    :param df_dict: dict, dict of dataframes from different atomic actions
+    """
+    for atomic_action in df_dict:
+        df_dict[atomic_action].to_csv("{}/{}.csv".format(csv_files_path, atomic_action))
+        print("{}/{}.csv created succesfully".format(csv_files_path, atomic_action))
+
+def push_to_gsheet(df_dict, google_svc_acc_key, email_id):
     """Push csv file to Google Sheets
-    :param csv_file_path: str, path to csv file
+    :param df_dict: dict, dict of dataframes from different atomic actions
     :param google_svc_acc_key: str, path to json credentials for google service account
     :param email_id: str, email id of user who will be given edit access to google sheet
     """
@@ -85,36 +106,40 @@ def push_to_gsheet(csv_file_path, google_svc_acc_key, email_id):
     gc = gspread.authorize(credentials)
 
     sh = gc.create(params.sheetname)
+
+    for atomic_action in df_dict:
+        ws = sh.add_worksheet(atomic_action, rows=len(df_dict[atomic_action]), cols=2)
+        set_with_dataframe(ws, df_dict[atomic_action])
+        format_cell_range(ws, "1:1000", fmt)
+        set_column_width(ws, "A", 290)
+        set_column_width(ws, "B", 190)
+
     sh.share(email_id, perm_type="user", role="writer")
-    spreadsheet_id = sh.id
     spreadsheet_url = f"https://docs.google.com/spreadsheets/d/{sh.id}"
-    with open(csv_file_path, "r") as f:
-        gc.import_csv(spreadsheet_id, f.read())
-    worksheet = sh.get_worksheet(0)
-    format_cell_range(worksheet, "1:1000", fmt)
-    set_column_width(worksheet, "A", 290)
-    set_column_width(worksheet, "B", 190)
-    set_column_width(worksheet, "C", 400)
     print(f"Google Spreadsheet link -> {spreadsheet_url}\n")
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "-c", "--csvfilepath", help="Path to create CSV file",
+        '-c', '--generatecsvfiles', action='store_true', required=False,
-        nargs='?', required=True, dest="csvfilepath"
+        help='Flag to enable csv file generation', dest="generatecsvfiles"
+    )
+    parser.add_argument(
+        "-f", "--csvfilespath", help="Path to directory to create CSV files",
+        nargs='?', required=False, dest="csvfilespath"
     )
     parser.add_argument(
         "-j", "--jsonfilepath", help="Rally JSON file path to scrape output from",
         nargs='?', required=True, dest="jsonfilepath",
     )
     parser.add_argument(
-        "-a", "--atomicaction", help="Atomic action to generate duration data",
+        "-a", "--atomicactions", help="Atomic actions to generate duration data",
-        nargs='?', required=True, dest="atomicaction",
+        nargs='+', required=True, dest="atomicactions",
     )
     parser.add_argument(
         '-g', '--uploadtogooglesheets', action='store_true', required=False,
-        help='Flag to enable csv file upload to Google Sheets', dest="uploadtogooglesheets"
+        help='Flag to enable upload to Google Sheets', dest="uploadtogooglesheets"
     )
     parser.add_argument(
         "-n", "--sheetname", help="Google Sheet name",
@@ -130,6 +155,12 @@ if __name__ == "__main__":
     )
     params = parser.parse_args()
 
+    if(params.generatecsvfiles and params.csvfilespath is None):
+        parser.error("{} requires {}.".format("--generatecsvfiles",
+                                              "--csvfilespath"))
+
+    df_dict = {}
+
     if(params.uploadtogooglesheets and
        (params.sheetname is None or params.googlesvcacckey is None or params.emailid is None)):
         parser.error("{} requires {} and {} and {}.".format("--uploadtogooglesheets",
@@ -137,7 +168,12 @@ if __name__ == "__main__":
                                                             "--googlesvcacckey",
                                                             "--emailid"))
 
-    convert_rally_json_to_csv(params.jsonfilepath, params.atomicaction,
+    for atomic_action in params.atomicactions:
-                              params.csvfilepath)
+        add_atomic_action_data_to_df(params.jsonfilepath, atomic_action,
+                                     df_dict)
+
+    if params.generatecsvfiles:
+        generate_csv_from_df(params.csvfilespath, df_dict)
+
     if params.uploadtogooglesheets:
-        push_to_gsheet(params.csvfilepath, params.googlesvcacckey, params.emailid)
+        push_to_gsheet(df_dict, params.googlesvcacckey, params.emailid)