diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 2bbd2176e..0eed789d0 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -343,7 +343,7 @@ Generate CSV file/Google Sheets from Rally json file -------------------------------------------- Rally generates a json file with data about atomic actions duration from each iteration. These atomic actions often occur multiple times within one iteration. Browbeat has a script which allows a user to generate a CSV file and also has an option to generate a Google Sheet about individual resource -duration through the Rally json file. To use the script to upload the CSV file to Google Sheets, a Google Drive service account is required. +duration through the Rally json file. To use the script to upload to Google Sheets, a Google Drive service account is required. The script sends an email to the email id of the user with the Google sheet if the --uploadgooglesheet option is enabled. To generate only a CSV file and not upload to Google Sheets : @@ -351,11 +351,27 @@ To generate only a CSV file and not upload to Google Sheets : :: $ source .browbeat-venv/bin/activate && cd utils - $ python rally_google_sheet_gen.py -c -j -a + $ python rally_google_sheet_gen.py -c -f + -j + -a + +To only upload to Google Sheets and not generate CSV files : + +:: + $ source .browbeat-venv/bin/activate && cd utils + $ python rally_google_sheet_gen.py + -j + -a -g + -s + -e -n + To generate a CSV file and upload to Google Sheets : :: $ source .browbeat-venv/bin/activate && cd utils - $ python rally_google_sheet_gen.py -c -j -a - -g -s -e -n + $ python rally_google_sheet_gen.py -c -f + -j + -a -g + -s + -e -n diff --git a/requirements.txt b/requirements.txt index 08e02525a..15ea5e4fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,5 @@ pykwalify>=1.8.0;python_version>='3' gspread oauth2client gspread_formatting +pandas +gspread_dataframe diff --git a/utils/rally_google_sheet_gen.py b/utils/rally_google_sheet_gen.py index 13fb48a3a..d88ba1b8f 100755 --- a/utils/rally_google_sheet_gen.py +++ b/utils/rally_google_sheet_gen.py @@ -20,57 +20,78 @@ # The script sends an email to the email id of the user with the Google sheet # if the --uploadgooglesheet option is enabled. -# To generate only a CSV file and not upload to Google Sheets : +# To generate only CSV files and not upload to Google Sheets : # $ source .browbeat-venv/bin/activate && cd utils -# $ python rally_google_sheet_gen.py -c -# -j -a +# $ python rally_google_sheet_gen.py -c -f +# -j +# -a + +# To only upload to Google Sheets and not generate CSV files : + +# $ source .browbeat-venv/bin/activate && cd utils +# $ python rally_google_sheet_gen.py +# -j +# -a -g +# -s +# -e -n # To generate a CSV file and upload to Google Sheets : # $ source .browbeat-venv/bin/activate && cd utils -# $ python rally_google_sheet_gen.py -c -# -j -a -g +# $ python rally_google_sheet_gen.py -c -f +# -j +# -a -g # -s # -e -n import json -import csv +import pandas as pd import gspread from oauth2client.service_account import ServiceAccountCredentials from gspread_formatting import cellFormat from gspread_formatting import format_cell_range from gspread_formatting import set_column_width +from gspread_dataframe import set_with_dataframe import argparse -def convert_rally_json_to_csv(json_file_path, atomic_action, csv_file_path): +def add_atomic_action_data_to_df(json_file_path, atomic_action, df_dict): """Convert rally json data to csv file containing atomic action data :param json_file_path: str, path to rally json file :param atomic_action: str, atomic action to generate duration data - :param csv_file_path: str, path to write csv file + :param df_dict: dict, dict of dataframes from different atomic actions """ json_file = open(json_file_path) json_data = json.load(json_file) - with open(csv_file_path, 'w', newline='') as csvfile: - csvwriter = csv.writer(csvfile, delimiter=',', - quotechar='|', quoting=csv.QUOTE_MINIMAL) + df = pd.DataFrame({"Resource Number": [], "Duration(in seconds)": []}) - csvwriter.writerow(["Resource Number", "Duration(in seconds)"]) - resource_number = 1 + resource_number = 1 - for iteration in json_data[0]["result"]: - for atomic_action_json in iteration["atomic_actions"]: - if atomic_action in atomic_action_json: - atomic_action_duration = iteration["atomic_actions"][atomic_action_json] - csvwriter.writerow([resource_number, atomic_action_duration]) - resource_number += 1 + for iteration in json_data[0]["result"]: + for atomic_action_json in iteration["atomic_actions"]: + if atomic_action in atomic_action_json: + atomic_action_duration = iteration["atomic_actions"][atomic_action_json] + df = df.append({"Resource Number": resource_number, + "Duration(in seconds)": atomic_action_duration}, + ignore_index=True) + resource_number += 1 - print("CSV file {} generated successfully.".format(csv_file_path)) + df_dict[atomic_action] = df + print("Pandas DF for atomic action {} generated successfully.".format(atomic_action)) -def push_to_gsheet(csv_file_path, google_svc_acc_key, email_id): +def generate_csv_from_df(csv_files_path, df_dict): + """Generate csv files from pandas dataframe + :param csv_files_path: str, path of directory to generate csv files + :param df_dict: dict, dict of dataframes from different atomic actions + """ + for atomic_action in df_dict: + df_dict[atomic_action].to_csv("{}/{}.csv".format(csv_files_path, atomic_action)) + print("{}/{}.csv created succesfully".format(csv_files_path, atomic_action)) + +def push_to_gsheet(df_dict, google_svc_acc_key, email_id): """Push csv file to Google Sheets - :param csv_file_path: str, path to csv file + :param df_dict: dict, dict of dataframes from different atomic actions :param google_svc_acc_key: str, path to json credentials for google service account :param email_id: str, email id of user who will be given edit access to google sheet """ @@ -85,36 +106,40 @@ def push_to_gsheet(csv_file_path, google_svc_acc_key, email_id): gc = gspread.authorize(credentials) sh = gc.create(params.sheetname) + + for atomic_action in df_dict: + ws = sh.add_worksheet(atomic_action, rows=len(df_dict[atomic_action]), cols=2) + set_with_dataframe(ws, df_dict[atomic_action]) + format_cell_range(ws, "1:1000", fmt) + set_column_width(ws, "A", 290) + set_column_width(ws, "B", 190) + sh.share(email_id, perm_type="user", role="writer") - spreadsheet_id = sh.id spreadsheet_url = f"https://docs.google.com/spreadsheets/d/{sh.id}" - with open(csv_file_path, "r") as f: - gc.import_csv(spreadsheet_id, f.read()) - worksheet = sh.get_worksheet(0) - format_cell_range(worksheet, "1:1000", fmt) - set_column_width(worksheet, "A", 290) - set_column_width(worksheet, "B", 190) - set_column_width(worksheet, "C", 400) print(f"Google Spreadsheet link -> {spreadsheet_url}\n") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( - "-c", "--csvfilepath", help="Path to create CSV file", - nargs='?', required=True, dest="csvfilepath" + '-c', '--generatecsvfiles', action='store_true', required=False, + help='Flag to enable csv file generation', dest="generatecsvfiles" + ) + parser.add_argument( + "-f", "--csvfilespath", help="Path to directory to create CSV files", + nargs='?', required=False, dest="csvfilespath" ) parser.add_argument( "-j", "--jsonfilepath", help="Rally JSON file path to scrape output from", nargs='?', required=True, dest="jsonfilepath", ) parser.add_argument( - "-a", "--atomicaction", help="Atomic action to generate duration data", - nargs='?', required=True, dest="atomicaction", + "-a", "--atomicactions", help="Atomic actions to generate duration data", + nargs='+', required=True, dest="atomicactions", ) parser.add_argument( '-g', '--uploadtogooglesheets', action='store_true', required=False, - help='Flag to enable csv file upload to Google Sheets', dest="uploadtogooglesheets" + help='Flag to enable upload to Google Sheets', dest="uploadtogooglesheets" ) parser.add_argument( "-n", "--sheetname", help="Google Sheet name", @@ -130,6 +155,12 @@ if __name__ == "__main__": ) params = parser.parse_args() + if(params.generatecsvfiles and params.csvfilespath is None): + parser.error("{} requires {}.".format("--generatecsvfiles", + "--csvfilespath")) + + df_dict = {} + if(params.uploadtogooglesheets and (params.sheetname is None or params.googlesvcacckey is None or params.emailid is None)): parser.error("{} requires {} and {} and {}.".format("--uploadtogooglesheets", @@ -137,7 +168,12 @@ if __name__ == "__main__": "--googlesvcacckey", "--emailid")) - convert_rally_json_to_csv(params.jsonfilepath, params.atomicaction, - params.csvfilepath) + for atomic_action in params.atomicactions: + add_atomic_action_data_to_df(params.jsonfilepath, atomic_action, + df_dict) + + if params.generatecsvfiles: + generate_csv_from_df(params.csvfilespath, df_dict) + if params.uploadtogooglesheets: - push_to_gsheet(params.csvfilepath, params.googlesvcacckey, params.emailid) + push_to_gsheet(df_dict, params.googlesvcacckey, params.emailid)