Error handling and debugging enhancements

1. Use back-ported Python 3.x functions for multi threading VM creation;
2. Show NOVA debug logs when --debug is passed;
3. Don't run test cases if VM creation fails;
4. User-friendly display for Ctrl-C;
5. Replace force_delete() to delete() for deleting VMs;

Change-Id: I380afac87c873e56148a80010dd4a21abed8e4eb

kloudbuster/base_compute.py

@@ -111,7 +111,9 @@ class BaseCompute(object):
 
     def detach_vol(self):
         if self.instance and self.vol:
-            self.novaclient.volumes.delete_server_volume(self.instance.id, self.vol.id)
+            attached_vols = self.novaclient.volumes.get_server_volumes(self.instance.id)
+            if len(attached_vols):
+                self.novaclient.volumes.delete_server_volume(self.instance.id, self.vol.id)
 
     def find_image(self, image_name):
         """

kloudbuster/base_network.py

@@ -167,7 +167,7 @@ class BaseNetwork(object):
         Security groups, keypairs and instances
         """
         flag = True
-        if self.instance_list[0].vol:
+        if self.instance_list and self.instance_list[0].vol:
             bs_obj = base_storage.BaseStorage(self.cinder_client)
 
         # Delete the instances first

kloudbuster/force_cleanup.py

@@ -246,7 +246,7 @@ class ComputeCleaner(AbstractCleaner):
                             fip_id = [x['id'] for x in fip_lst if x['floating_ip_address'] == fip]
                             self.neutron_client.delete_floatingip(fip_id[0])
                             self.report_deletion('FLOATING IP', fip)
-                        self.nova_client.servers.force_delete(id)
+                        self.nova_client.servers.delete(id)
                 except NotFound:
                     deleting_instances.remove(id)
                     self.report_not_found('INSTANCE', name)

kloudbuster/kloudbuster.py

@@ -13,8 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from concurrent.futures import ThreadPoolExecutor
 import json
-from multiprocessing.pool import ThreadPool
 import os
 import sys
 import threading
@@ -77,11 +77,12 @@ class Kloud(object):
         else:
             self.prefix = 'KBs'
             self.name = 'Server Kloud'
-        LOG.info("Creating kloud: " + self.prefix)
-
         # pre-compute the placement az to use for all VMs
         self.placement_az = scale_cfg['availability_zone'] \
             if scale_cfg['availability_zone'] else None
+        self.exc_info = None
+
+        LOG.info("Creating kloud: " + self.prefix)
         if self.placement_az:
             LOG.info('%s Availability Zone: %s' % (self.name, self.placement_az))
 
@@ -181,7 +182,8 @@ class Kloud(object):
         LOG.info("Creating Instance: " + instance.vm_name)
         instance.create_server(**instance.boot_info)
         if not instance.instance:
-            raise KBVMCreationException()
+            raise KBVMCreationException(
+                'Instance %s takes too long to become ACTIVE.' % instance.vm_name)
 
         if instance.vol:
             instance.attach_vol()
@@ -205,9 +207,12 @@ class Kloud(object):
             instance.ssh_ip = instance.fixed_ip
 
     def create_vms(self, vm_creation_concurrency):
-        tpool = ThreadPool(processes=vm_creation_concurrency)
-        for _ in tpool.imap(self.create_vm, self.get_all_instances()):
-            self.vm_up_count += 1
+        try:
+            with ThreadPoolExecutor(max_workers=vm_creation_concurrency) as executor:
+                for feature in executor.map(self.create_vm, self.get_all_instances()):
+                    self.vm_up_count += 1
+        except Exception:
+            self.exc_info = sys.exc_info()
 
 
 class KloudBuster(object):
@@ -267,7 +272,8 @@ class KloudBuster(object):
         creden_nova['auth_url'] = cred_dict['auth_url']
         creden_nova['project_id'] = cred_dict['tenant_name']
         creden_nova['version'] = 2
-        nova_client = novaclient(**creden_nova)
+        nova_client = novaclient(endpoint_type='publicURL',
+                                 http_log_debug=True, **creden_nova)
         for hypervisor in nova_client.hypervisors.list():
             if vars(hypervisor)['status'] == 'enabled':
                 ret_list.append(vars(hypervisor)['hypervisor_hostname'])
@@ -283,7 +289,8 @@ class KloudBuster(object):
         creden_nova['auth_url'] = cred_dict['auth_url']
         creden_nova['project_id'] = cred_dict['tenant_name']
         creden_nova['version'] = 2
-        nova_client = novaclient(**creden_nova)
+        nova_client = novaclient(endpoint_type='publicURL',
+                                 http_log_debug=True, **creden_nova)
         for az in nova_client.availability_zones.list():
             zoneName = vars(az)['zoneName']
             isAvail = vars(az)['zoneState']['available']
@@ -413,6 +420,8 @@ class KloudBuster(object):
             LOG.error(e.message)
         except Exception:
             traceback.print_exc()
+        except KeyboardInterrupt:
+            LOG.info('Terminating KloudBuster...')
         finally:
             if self.server_cfg['cleanup_resources'] and self.client_cfg['cleanup_resources']:
                 self.cleanup()
@@ -507,6 +516,12 @@ class KloudBuster(object):
             self.server_vm_create_thread.join()
             self.client_vm_create_thread.join()
 
+        if self.testing_kloud and self.testing_kloud.exc_info:
+            raise self.testing_kloud.exc_info[1], None, self.testing_kloud.exc_info[2]
+
+        if self.kloud and self.kloud.exc_info:
+            raise self.kloud.exc_info[1], None, self.kloud.exc_info[2]
+
         # Function that print all the provisioning info
         self.print_provision_info()
 

kloudbuster/log.py

@@ -48,7 +48,7 @@ def setup(product_name, logfile=None):
     CONF.logging_default_format_string = '%(asctime)s %(levelname)s %(message)s'
 
     oslogging.setup(CONF, product_name)
-    # Adding the FileHandler to all known loggers inside KloudBuster
+    # Adding FileHandler to KloudBuster if logfile is supplied
     if logfile:
         if os.path.exists(logfile):
             os.remove(logfile)
@@ -62,7 +62,6 @@ def setup(product_name, logfile=None):
             project=product_name).logger.setLevel(logging.KBDEBUG)
 
 def getLogger(name="unknown", version="unknown"):
-
     if name not in oslogging._loggers:
         oslogging._loggers[name] = KloudBusterContextAdapter(
             logging.getLogger(name), {"project": "kloudbuster",

kloudbuster/users.py

@@ -20,7 +20,7 @@ from cinderclient.v2 import client as cinderclient
 from keystoneclient import exceptions as keystone_exception
 import log as logging
 from neutronclient.v2_0 import client as neutronclient
-from novaclient.client import Client
+from novaclient import client as novaclient
 
 LOG = logging.getLogger(__name__)
 
@@ -207,7 +207,8 @@ class User(object):
         creden_nova['project_id'] = self.tenant.tenant_name
         creden_nova['version'] = 2
 
-        self.nova_client = Client(endpoint_type='publicURL', **creden_nova)
+        self.nova_client = novaclient.Client(endpoint_type='publicURL',
+                                             http_log_debug=True, **creden_nova)
         self.cinder_client = cinderclient.Client(endpoint_type='publicURL', **creden_nova)
 
         if self.tenant.kloud.reusing_tenants:

requirements.txt

@@ -5,6 +5,7 @@
 pbr>=1.3
 Babel>=1.3
 
+futures>=3.0.5
 python-openstackclient>=2.2.0
 python-neutronclient>=4.0.0
 attrdict>=2.0.0