From 13b14c575409ae5f9a2f8cc9cae5295ee51e080b Mon Sep 17 00:00:00 2001 From: Megan Guiney Date: Sat, 1 Apr 2017 23:09:33 -0700 Subject: [PATCH] Add a script to update the internal RefStack DB using a provided csv. See the README for more information on how to use this script. Change-Id: I077d8588b69e2dd0c6302557a2b2867ca4bc1cc1 --- tools/update-rs-db.py | 131 +++++++++++++++++++++++++++++++++++++++++ tools/update-rs-db.rst | 57 ++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100755 tools/update-rs-db.py create mode 100644 tools/update-rs-db.rst diff --git a/tools/update-rs-db.py b/tools/update-rs-db.py new file mode 100755 index 00000000..43e91c9a --- /dev/null +++ b/tools/update-rs-db.py @@ -0,0 +1,131 @@ +#!/usr/bin/python +import requests +import argparse +import os +import json + + +def getData(entry): + guidelines = ["2015.03", "2015.04", "2015.05", "2015.07", "2016.01", + "2016.08", "2017.01"] + components = ["Platform", "Compute", "Storage"] + if len(entry) < 10: + return None, None, None + if entry[9] != "" and entry[9] != " ": + refstackLink = entry[9] + testId = refstackLink.split("/")[-1] + else: + refstackLink = None + testId = None + if entry[4] != "" and entry[4] != " " and entry[4] in guidelines: + guideline = entry[4] + else: + guideline = None + if entry[5] != "" and entry[5] != " " and entry[5] in components: + target = entry[5].lower() + if target == "storage": + target = "object" + else: + target = None + return testId, guideline, target + + +def linkChk(link, token): + print("checking result with a test ID of: " + link.split("/")[-1]) + if not link: + return False + try: + if " " in link: + return False + response = requests.get( + link, headers={'Authorization': 'Bearer ' + token}) + if response.status_code == 200: + return json.loads(response.text) + else: + print("Link check response_status_code=" + + str(response.status_code)) + return False + except requests.exceptions as err: + print(err) + return False + + +def updateResult(apiLink, target, guideline, token): + response = requests.post(apiLink + '/meta/shared', headers={ + 'Authorization': 'Bearer ' + token}, data='true') + if response.status_code != 201: + print("Update shared status response_status_code=" + + str(response.status_code)) + return False + if ".json" not in guideline: + guideline = str(guideline) + ".json" + response = requests.post(apiLink + '/meta/guideline', headers={ + 'Authorization': 'Bearer ' + token}, data=guideline) + if response.status_code != 201: + print("Update guideline response_status_code=" + + str(response.status_code)) + return False + response = requests.post(apiLink + '/meta/target', headers={ + 'Authorization': 'Bearer ' + token}, data=target) + if response.status_code != 201: + print("Update target response_status_code=" + + str(response.status_code)) + return False + print("test result updated. Verifying.") + response = requests.put(apiLink, headers={ + 'Authorization': 'Bearer ' + token}, json={'verification_status': 1}) + if response.status_code != 201: + return False + print("Test result verified.") + return True + + +def main(): + linect = 0 + parser = argparse.ArgumentParser( + "Update the internal RefStack db using a csv file") + parser.add_argument("--file", "-f", metavar='f', type=str, action="store", + required=True, + help="csv source for the data to use in updates") + parser.add_argument( + "--endpoint", "-e", metavar='e', + type=str, action="store", required=True, + help="the base URL of the endpoint. ex: http://examplerefstack.com/v1") + parser.add_argument("--token", "-t", metavar="t", type=str, + action="store", required=True, help="API auth token") + result = parser.parse_args() + infile = result.file + endpoint = result.endpoint + token = result.token + with open(infile) as f: + for line in f: + linect = linect + 1 + entry = line.split(",") + testId, guideline, target = getData(entry) + if testId is None or guideline is None or target is None: + print( + "entry found at line " + str(linect) + + " cannot be updated and verified: entry incomplete.") + else: + apiLink = os.path.join(endpoint, 'results', testId) + testResult = linkChk(apiLink, token) + if testResult: + if testResult.get('verification_status'): + print( + "Result has already been verified; nothing to do.") + else: + print( + "Result link is valid. Updating result with ID " + + testId) + success = updateResult(apiLink, target, guideline, + token) + if not success: + print("update of the results with the ID " + + testId + " failed. please recheck your " + + "spreadsheet and try again") + else: + print("the test result " + testId + + " cannot be verified due to a broken result link.") + + +main() diff --git a/tools/update-rs-db.rst b/tools/update-rs-db.rst new file mode 100644 index 00000000..bde444a2 --- /dev/null +++ b/tools/update-rs-db.rst @@ -0,0 +1,57 @@ +####################################################################### +# update-rs-db.py # +####################################################################### + +This document contains some details that are neccessary to know to be +successful in the usage of the script update-rs-db.py. + +The script can be run using the following formatting: +"./update-rs-db.py --file /tmp/datasource.csv --endpoint +http://example.com:8000/v1 --token " + +This script updates RefStack tests as verified given a specific +spreadsheet. The columns in this spreadsheet are, in this order: + - Company Name + - Product Name + - Type (Distribution, Public, or Private) + - Region + - Guideline + - Component (Compute, Platform, or Object) + - Reported Release + - Passed Release + - Federated identity (yes/no) + - Refstack Link + - Zendesk Link + - Marketplace Link + - License Date + - Update Product (yes/no) + - Contacts + - Notes + - License Link + - Active (1 or 0) + - Public (1 or 0) + +The data is pulled from a csv file. The default csv name is toadd.csv, +but using the -f flag, we can use csv of a different filename. + +The refstack database that we are pushing updates to is set via the "-e", +or "--endpoint flag. This flag specifies the refstack api endpoint to be +used to update the database. This is a required flag. + +Because editing arbitrary test results requires administrative privileges, +an auth token must be used with the RefStack API. This token can be +generated by entering the command "jwt --key="$( cat +)" --alg=RS256 user_openid= exp=+100500". This generates a +json web token, which we must link using the "-t" or "--token" flag. Because +we cannot auth without this token, the token is a required flag. + +The script will go through each line of the CSV, grabbing the refstack link, +the guideline, and the component. It also uses the refstack result to get a +test result Id. + +It then uses that test ID to update the internal db using refstack's built +in RESTful api. + +Lastly, if at least one of the links has proven to be valid, we will +then use the same RESTful api, and test ID to update the verification_status +field associated with that test result.