Add discovery methods.

Add methods to discover tenants and identity region name.

Change-Id: I7e711663c80a5d181031d4c2e0c5338f12128263
Partially-implements: blueprint discover-cloud-artifacts

config.json.sample

@@ -8,13 +8,6 @@
 
     "tempest_config":
     {
-        "identity":
-        {
-            "region": "RegionOne",
-            "tenant_name": "demo",
-            "alt_tenant_name": "alt_demo",
-            "admin_tenant_name": "admin"
-        },
         "compute":
         {
             "image_ref": "a8d70acb-f1c4-4171-b0ce-d73e5de21a9d",

refstack/tools/execute_test.py

@@ -32,11 +32,11 @@ class Test:
     def __init__(self, args):
         '''Prepare a tempest test against a cloud.'''
 
-        _format = "%(asctime)s %(name)s %(levelname)s %(message)s"
+        log_format = "%(asctime)s %(name)s %(levelname)s %(message)s"
         if args.verbose:
-            logging.basicConfig(level=logging.INFO, format=_format)
+            logging.basicConfig(level=logging.INFO, format=log_format)
         else:
-            logging.basicConfig(level=logging.CRITICAL, format=_format)
+            logging.basicConfig(level=logging.CRITICAL, format=log_format)
         self.logger = logging.getLogger("execute_test")
 
         self.app_server_address = None
@@ -44,61 +44,66 @@ class Test:
         if args.callback:
             self.app_server_address, self.test_id = args.callback
 
-        self.extraConfDict = dict()
+        self.mini_conf_dict = json.loads(self.get_mini_config())
+        self.extra_conf_dict = dict()
         if args.conf_json:
-            self.extraConfDict = args.conf_json
+            self.extra_conf_dict = args.conf_json
 
         self.testcases = {"testcases": ["tempest"]}
         if args.testcases:
             self.testcases = {"testcases": args.testcases}
 
-        self.tempestHome = os.path.join(os.path.dirname(
+        self.tempest_home =\
-                                        os.path.abspath(__file__)),
+            os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tempest')
-                                        'tempest')
+
         if args.tempest_home:
-            self.tempestHome = args.tempest_home
+            self.tempest_home = args.tempest_home
 
-        self.sampleConfFile = os.path.join(self.tempestHome, 'etc',
+        self.sample_conf_file = os.path.join(self.tempest_home, 'etc',
-                                           'tempest.conf.sample')
+                                             'tempest.conf.sample')
-        self.tempestConfFile = os.path.join(self.tempestHome, 'tempest.config')
+        self.tempest_conf_file = os.path.join(self.tempest_home,
-        self.resultDir = os.path.join(self.tempestHome, '.testrepository')
+                                              'tempest.config')
-        self.result = os.path.join(self.resultDir, 'result')
+        self.result_dir = os.path.join(self.tempest_home, '.testrepository')
-        self.tempestScript = os.path.join(self.tempestHome, 'run_tests.sh')
+        self.result = os.path.join(self.result_dir, 'result')
-        self.sampleConfParser = ConfigParser.SafeConfigParser()
+        self.tempest_script = os.path.join(self.tempest_home, 'run_tests.sh')
-        self.sampleConfParser.read(self.sampleConfFile)
+        self.sample_conf_parser = ConfigParser.SafeConfigParser()
+        self.sample_conf_parser.read(self.sample_conf_file)
 
-    def genConfig(self):
+    def gen_config(self):
         '''Merge mini config, extra config, tempest.conf.sample
            and write to tempest.config.
         '''
         self.logger.info('Generating tempest.config')
-        miniConfDict = json.loads(self.getMiniConfig())
+        self.merge_to_sample_conf(self.mini_conf_dict)
-        self.mergeToSampleConf(miniConfDict)
+        self.merge_to_sample_conf(self.extra_conf_dict)
-        self.mergeToSampleConf(self.extraConfDict)
+        #discovered config will not overwrite the value in the
-        self.sampleConfParser.write(open(self.tempestConfFile, 'w'))
+        #mini_conf_dict and extra_conf_dict
+        discovered_conf_dict = self._build_discovered_dict_conf()
+        self.merge_to_sample_conf(discovered_conf_dict)
+        self.sample_conf_parser.write(open(self.tempest_conf_file, 'w'))
 
-    def mergeToSampleConf(self, dic):
+    def merge_to_sample_conf(self, dic):
         '''Merge values in a dictionary to tempest.conf.sample.'''
         for section, data in dic.items():
             for key, value in data.items():
-                if self.sampleConfParser.has_option(section, key):
+                if self.sample_conf_parser.has_option(section, key):
-                    self.sampleConfParser.set(section, key, value)
+                    self.sample_conf_parser.set(section, key, value)
 
-    def getMiniConfig(self):
+    def get_mini_config(self):
         '''Return a mini config in JSON string.'''
         if self.app_server_address and self.test_id:
             url = "http://%s/get-miniconf?test_id=%s" % \
                 (self.app_server_address, self.test_id)
             try:
-                j = urllib2.urlopen(url=url, timeout=10)
+                req = urllib2.urlopen(url=url, timeout=10)
-                return j.readlines()[0]
+                return req.readlines()[0]
             except:
                 self.logger.critical('Failed to get mini config from %s' % url)
                 raise
         else:
             return json.dumps(dict())
 
-    def getTestCases(self):
+    def get_test_cases(self):
         '''Return list of tempest testcases in JSON string.
 
            For certification, the list will contain only one test case.
@@ -109,33 +114,34 @@ class Test:
             url = "http://%s/get-testcases?test_id=%s" % \
                 (self.app_server_address, self.test_id)
             try:
-                j = urllib2.urlopen(url=url, timeout=10)
+                req = urllib2.urlopen(url=url, timeout=10)
-                return j.readlines()[0]
+                return req.readlines()[0]
             except:
                 self.logger.crtical('Failed to get test cases from %s' % url)
                 raise
         else:
             return json.dumps(self.testcases)
 
-    def runTestCases(self):
+    def run_test_cases(self):
         '''Executes each test case in the testcase list.'''
 
         #Make a backup in case previous data exists in the the directory
-        if os.path.exists(self.resultDir):
+        if os.path.exists(self.result_dir):
             date = time.strftime("%m%d%H%M%S")
-            backupPath = os.path.join(os.path.dirname(self.resultDir),
+            backup_path = os.path.join(os.path.dirname(self.result_dir),
-                                      "%s_backup_%s" %
+                                       "%s_backup_%s" %
-                                      (os.path.basename(self.resultDir), date))
+                                       (os.path.basename(self.result_dir),
+                                        date))
             self.logger.info("Rename existing %s to %s" %
-                             (self.resultDir, backupPath))
+                             (self.result_dir, backup_path))
-            os.rename(self.resultDir, backupPath)
+            os.rename(self.result_dir, backup_path)
 
         #Execute each testcase.
-        testcases = json.loads(self.getTestCases())['testcases']
+        testcases = json.loads(self.get_test_cases())['testcases']
         self.logger.info('Running test cases')
         for case in testcases:
             cmd = ('%s -C %s -N -- %s' %
-                   (self.tempestScript, self.tempestConfFile, case))
+                   (self.tempest_script, self.tempest_conf_file, case))
             #When a testcase fails
             #continue execute all remaining cases so any partial result can be
             #reserved and posted later.
@@ -145,7 +151,7 @@ class Test:
                 self.logger.error('%s %s testcases failed to complete' %
                                   (e, case))
 
-    def postTestResult(self):
+    def post_test_result(self):
         '''Post the combined results back to the server.'''
         if self.app_server_address and self.test_id:
             self.logger.info('Send back the result')
@@ -160,70 +166,231 @@ class Test:
         else:
             self.logger.info('Testr result can be found at %s' % (self.result))
 
-    def combineTestrResult(self):
+    def combine_test_result(self):
         '''Generate a combined testr result.'''
-        r_list = [l for l in os.listdir(self.resultDir)
+        testr_results = [item for item in os.listdir(self.result_dir)
-                  if fnmatch.fnmatch(l, '[0-9]*')]
+                         if fnmatch.fnmatch(item, '[0-9]*')]
-        r_list.sort(key=int)
+        testr_results.sort(key=int)
         with open(self.result, 'w') as outfile:
-            for r in r_list:
+            for fp in testr_results:
-                with open(os.path.join(self.resultDir, r), 'r') as infile:
+                with open(os.path.join(self.result_dir, fp), 'r') as infile:
                     outfile.write(infile.read())
         self.logger.info('Combined testr result')
 
     def run(self):
         '''Execute tempest test against the cloud.'''
 
-        self.genConfig()
+        self.gen_config()
 
-        self.runTestCases()
+        self.run_test_cases()
 
-        self.combineTestrResult()
+        self.combine_test_result()
 
-        self.postTestResult()
+        self.post_test_result()
+
+    # These methods are for identity discovery
+    def _subtract_dictionaries(self, discovered_conf_dict, conf_dict):
+        '''Remove the configs in conf_dict from discovered_conf_dict.'''
+        for section, data in discovered_conf_dict.items():
+            for key in data.keys():
+                if section in conf_dict and key in conf_dict[section]:
+                    self.logger.info("will not discover [%s] %s because caller"
+                                     " chose to overwrite." % (section, key))
+                    del discovered_conf_dict[section][key]
+
+    def _build_discovered_dict_conf(self):
+        '''Return discovered tempest configs in a json string.'''
+        self.logger.info("Starting tempest config discovery")
+
+        #This is the default discovery items
+        #in which the tempest.sample.conf will be discovered.
+        discovery_conf_dict =\
+            {"identity": {"region": self.get_identity_region,
+                          "admin_tenant_name": self.get_admin_tenant_name,
+                          "tenant_name": self.get_tenant_name,
+                          "alt_tenant_name": self.get_alt_tenant_name}
+             }
+
+        #Remove the configs from the default discovery
+        #for those that caller choose to overwrite.
+        self._subtract_dictionaries(discovery_conf_dict, self.mini_conf_dict)
+        self._subtract_dictionaries(discovery_conf_dict, self.extra_conf_dict)
+
+        #populate configs
+        for section, data in discovery_conf_dict.items():
+                for key in data.keys():
+                    discovery_conf_dict[section][key] =\
+                        discovery_conf_dict[section][key]()
+        self.logger.info("Discovered configs: %s" % discovery_conf_dict)
+        return discovery_conf_dict
+
+    def get_keystone_token(self, url, user, password, tenant=""):
+        ''' Returns the json response from keystone tokens API call.'''
+        parameter = {"auth": {"tenantName": tenant,
+                              "passwordCredentials":
+                              {"username": user,
+                               "password": password}
+                              }
+                     }
+        header = {"Content-type": "application/json"}
+        try:
+            req = requests.post(url, data=json.dumps(parameter),
+                                headers=header)
+        except:
+            self.logger.critical("failed to get a Keystone token for"
+                                 "url: %s user: %s tenant: %s"
+                                 (url, user, tenant))
+            raise
+
+        return req.content
+
+    def get_tenants(self, token_id):
+        '''Return a list of tenants of a token_id.'''
+        keystone_url = self.sample_conf_parser.get("identity", "uri")
+        headers = {"Content-type": "application/json",
+                   "X-Auth-Token": token_id}
+        try:
+            req = requests.get(keystone_url + "/tenants", headers=headers)
+        except:
+            self.logger.critical("failed to get tenant for token id %s"
+                                 "from %s" % (token_id, keystone_url))
+            raise
+        return json.loads(req.content)["tenants"]
+
+    def get_alt_tenant_name(self):
+        '''Return the alt_tenant_name
+        '''
+        keystone_url = self.sample_conf_parser.get("identity", "uri")
+        alt_user = self.sample_conf_parser.get("identity", "alt_username")
+        alt_pw = self.sample_conf_parser.get("identity", "alt_password")
+        token_id = json.loads(self.get_keystone_token(url=keystone_url +
+                                                      "/tokens",
+                                                      user=alt_user,
+                                                      password=alt_pw)
+                              )["access"]["token"]["id"]
+        '''TODO: Assuming the user only belongs to one tenant'''
+        try:
+            alt_tenant = self.get_tenants(token_id)[0]["name"]
+        except:
+            self.logger.critical("failed to get the tenant for alt_username %s"
+                                 "from %s" % (alt_user, keystone_url))
+            raise
+        return alt_tenant
+
+    def get_tenant_name(self):
+        '''Return the tenant_name.
+        '''
+        keystone_url = self.sample_conf_parser.get("identity", "uri")
+        user = self.sample_conf_parser.get("identity", "username")
+        pw = self.sample_conf_parser.get("identity", "password")
+        token_id = json.loads(self.get_keystone_token(url=keystone_url +
+                                                      "/tokens",
+                                                      user=user,
+                                                      password=pw)
+                              )["access"]["token"]["id"]
+        '''TODO: Assuming the user only belongs to one tenant'''
+        try:
+            tenant = self.get_tenants(token_id)[0]["name"]
+        except:
+            self.logger.critical("failed to get the tenant for username %s"
+                                 "from %s" % (user, keystone_url))
+            raise
+        return tenant
+
+    def get_admin_tenant_name(self):
+        '''
+        Return the admin_tenant_name.
+        TODO: save admin tenant as an attribute so get_identity_region()
+        method can directly use it.
+        '''
+        keystone_url = self.sample_conf_parser.get("identity", "uri")
+        admin_user = self.sample_conf_parser.get("identity", "admin_username")
+        admin_pw = self.sample_conf_parser.get("identity", "admin_password")
+        token_id = json.loads(self.get_keystone_token(url=keystone_url +
+                                                      "/tokens",
+                                                      user=admin_user,
+                                                      password=admin_pw)
+                              )["access"]["token"]["id"]
+
+        '''TODO: Authenticate as "admin" (public URL) against each tenant found
+        in tanantList until a tenant is found on which "admin" has
+        "admin"role. For now, assuming admin user ONLY belongs to admin tenant
+        and the admin has admin role as defined in
+        tempest.sample.conf.identiy.admin_role
+        '''
+        try:
+            tenant = self.get_tenants(token_id)[0]["name"]
+        except:
+            self.logger.critical("failed to get the tenant for"
+                                 "admin_username %s from %s" %
+                                 (admin_user, keystone_url))
+            raise
+        return tenant
+
+    def get_identity_region(self):
+        '''Return the identity region.
+        '''
+        keystone_url = self.sample_conf_parser.get("identity", "uri")
+        admin_user = self.sample_conf_parser.get("identity", "admin_username")
+        admin_pw = self.sample_conf_parser.get("identity", "admin_password")
+        admin_tenant = self.get_admin_tenant_name()
+        '''
+        TODO: Preserve the admin token id as an attribute because
+        the admin_token will be used to for image discovery
+        '''
+        admin_token = json.loads(self.get_keystone_token
+                                (url=keystone_url + "/tokens",
+                                 user=admin_user,
+                                 password=admin_pw,
+                                 tenant=admin_tenant))
+        '''TODO: assume there is only one identity endpoint'''
+        identity_region =\
+            [service["endpoints"][0]["region"]
+             for service in admin_token["access"]["serviceCatalog"]
+             if service["type"] == "identity"][0]
+
+        return identity_region
 
     ''' TODO: The remaining methods are for image discovery. '''
-
+    def create_image(self):
-    def createImage(self):
         '''Download and create cirros image.
            Return the image reference id
         '''
         pass
 
-    def findSmallestFlavor(self):
+    def find_smallest_flavor(self):
         '''Find the smallest flavor by sorting by memory size.
         '''
         pass
 
-    def deleteImage(self):
+    def delete_image(self):
         '''Delete a image.
         '''
         pass
 
 if __name__ == '__main__':
-    ''' Generate tempest.conf from a tempest.conf.sample and then run test
+    ''' Generate tempest.conf from a tempest.conf.sample and then run test.'''
-    '''
     parser = argparse.ArgumentParser(description='Starts a tempest test',
                                      formatter_class=argparse.
                                      ArgumentDefaultsHelpFormatter)
-    conflictGroup = parser.add_mutually_exclusive_group()
+    conflict_group = parser.add_mutually_exclusive_group()
 
-    conflictGroup.add_argument("--callback",
+    conflict_group.add_argument("--callback",
-                               nargs=2,
+                                nargs=2,
-                               metavar=("APP_SERVER_ADDRESS", "TEST_ID"),
+                                metavar=("APP_SERVER_ADDRESS", "TEST_ID"),
-                               type=str,
+                                type=str,
-                               help="refstack API IP address and test ID to\
+                                help="refstack API IP address and test ID to\
-                               retrieve configurations. i.e.:\
+                                       retrieve configurations. i.e.:\
-                               --callback 127.0.0.1:8000 1234")
+                                       --callback 127.0.0.1:8000 1234")
 
     parser.add_argument("--tempest-home",
                         help="tempest directory path")
 
     #with nargs, arguments are returned as a list
-    conflictGroup.add_argument("--testcases",
+    conflict_group.add_argument("--testcases",
-                               nargs='+',
+                                nargs='+',
-                               help="tempest test cases. Use space to separate\
+                                help="tempest test cases. Use space to\
-                               each testcase")
+                                separate each testcase")
     '''
     TODO: May need to decrypt/encrypt password in args.JSON_CONF
     '''