Task manager will now detect some creation errors and mark the instances.

reddwarf/instance/models.py

@@ -71,7 +71,7 @@ class InstanceStatus(object):
 
 def load_simple_instance_server_status(context, db_info):
     """Loads a server or raises an exception."""
-    if InstanceTasks.BUILDING == db_info.task_status:
+    if 'BUILDING' == db_info.task_status.action:
         db_info.server_status = "BUILD"
         db_info.addresses = {}
     else:
@@ -154,22 +154,28 @@ class SimpleInstance(object):
 
     @property
     def status(self):
-        #TODO(tim.simpson): As we enter more advanced cases dealing with
+        ### Check for taskmanager errors.
-        # timeouts determine if the task_status should be integrated here
+        if self.db_info.task_status.is_error:
-        # or removed entirely.
+            return InstanceStatus.ERROR
-        if InstanceTasks.BUILDING == self.db_info.task_status:
+
+        ### Check for taskmanager status.
+        ACTION = self.db_info.task_status.action
+        if 'BUILDING' == ACTION:
+            if 'ERROR' == self.db_info.server_status:
+                return InstanceStatus.ERROR
             return InstanceStatus.BUILD
-        if InstanceTasks.REBOOTING == self.db_info.task_status:
+        if 'REBOOTING' == ACTION:
             return InstanceStatus.REBOOT
-        if InstanceTasks.RESIZING == self.db_info.task_status:
+        if 'RESIZING' == ACTION:
             return InstanceStatus.RESIZE
 
-        # If the server is in any of these states they take precedence.
+        ### Check for server status.
         if self.db_info.server_status in ["BUILD", "ERROR", "REBOOT",
                                           "RESIZE"]:
             return self.db_info.server_status
 
-        if InstanceTasks.DELETING == self.db_info.task_status:
+        ### Report as Shutdown while deleting, unless there's an error.
+        if 'DELETING' == ACTION:
             if self.db_info.server_status in ["ACTIVE", "SHUTDOWN"]:
                 return InstanceStatus.SHUTDOWN
             else:
@@ -177,12 +183,14 @@ class SimpleInstance(object):
                     " status (%s).") % (self.id, self.db_info.server_status))
                 return InstanceStatus.ERROR
 
+        ### Check against the service status.
         # The service is only paused during a reboot.
         if ServiceStatuses.PAUSED == self.service_status.status:
             return InstanceStatus.REBOOT
         # If the service status is NEW, then we are building.
         if ServiceStatuses.NEW == self.service_status.status:
             return InstanceStatus.BUILD
+
         # For everything else we can look at the service status mapping.
         return self.service_status.status.api_status
 

reddwarf/instance/tasks.py

@@ -25,14 +25,16 @@ class InstanceTask(object):
     #                   once that revs up.
     _lookup = {}
 
-    def __init__(self, code, db_text):
+    def __init__(self, code, action, db_text, is_error=False):
         self._code = int(code)
+        self._action = action
         self._db_text = db_text
+        self._is_error = is_error
         InstanceTask._lookup[self._code] = self
 
     @property
-    def api_status(self):
+    def action(self):
-        return self._api_status
+        return self._action
 
     @property
     def code(self):
@@ -42,6 +44,10 @@ class InstanceTask(object):
     def db_text(self):
         return self._db_text
 
+    @property
+    def is_error(self):
+        return self._is_error
+
     def __eq__(self, other):
         if not isinstance(other, InstanceTask):
             return False
@@ -55,11 +61,18 @@ class InstanceTask(object):
 
 
 class InstanceTasks(object):
-    NONE = InstanceTask(0x01, 'NONE')
+    NONE = InstanceTask(0x01, 'NONE', 'No tasks for the instance.')
-    DELETING = InstanceTask(0x02, 'DELETING')
+    DELETING = InstanceTask(0x02, 'DELETING', 'Deleting the instance.')
-    REBOOTING = InstanceTask(0x03, 'REBOOTING')
+    REBOOTING = InstanceTask(0x03, 'REBOOTING', 'Rebooting the instance.')
-    RESIZING = InstanceTask(0x04, 'RESIZING')
+    RESIZING = InstanceTask(0x04, 'RESIZING', 'Resizing the instance.')
-    BUILDING = InstanceTask(0x05, 'BUILDING')
+    BUILDING = InstanceTask(0x05, 'BUILDING', 'The instance is building.')
+
+    BUILDING_ERROR_DNS = InstanceTask(0x50, 'BUILDING',
+        'Build error: DNS.', is_error=True)
+    BUILDING_ERROR_SERVER = InstanceTask(0x51, 'BUILDING',
+        'Build error: Server.', is_error=True)
+    BUILDING_ERROR_VOLUME = InstanceTask(0x52, 'BUILDING',
+        'Build error: Volume.', is_error=True)
 
 
 # Dissuade further additions at run-time.

reddwarf/taskmanager/models.py

@@ -50,20 +50,48 @@ class FreshInstanceTasks(FreshInstance):
 
     def create_instance(self, flavor_id, flavor_ram, image_id,
                         databases, users, service_type, volume_size):
+        volume_info = None
+        block_device_mapping = None
+        server = None
         try:
             volume_info = self._create_volume(volume_size)
             block_device_mapping = volume_info['block_device']
+        except Exception as e:
+            msg = "Error provisioning volume for instance."
+            err = inst_models.InstanceTasks.BUILDING_ERROR_VOLUME
+            self._log_and_raise(e, msg, err)
+
+        try:
             server = self._create_server(flavor_id, image_id, service_type,
-                                         block_device_mapping)
+                                     block_device_mapping)
             server_id = server.id
             # Save server ID.
             self.update_db(compute_instance_id=server_id)
+        except Exception as e:
+            msg = "Error creating server for instance."
+            err = inst_models.InstanceTasks.BUILDING_ERROR_SERVER
+            self._log_and_raise(e, msg, err)
+
+        try:
             self._create_dns_entry()
+        except Exception as e:
+            msg = "Error creating DNS entry for instance."
+            err = inst_models.InstanceTasks.BUILDING_ERROR_DNS
+            self._log_and_raise(e, msg, err)
+
+        if server:
             self._guest_prepare(server, flavor_ram, volume_info,
                                 databases, users)
-        finally:
+
+        if not self.db_info.task_status.is_error:
             self.update_db(task_status=inst_models.InstanceTasks.NONE)
 
+    def _log_and_raise(self, exc, message, task_status):
+        LOG.error(message)
+        LOG.error(exc)
+        self.update_db(task_status=task_status)
+        raise ReddwarfError(message=message)
+
     def _create_volume(self, volume_size):
         LOG.info("Entering create_volume")
         LOG.debug(_("Starting to create the volume for the instance"))

reddwarf/tests/fakes/nova.py

@@ -185,6 +185,8 @@ class FakeServers(object):
         server = FakeServer(self, self.context, id, name, image_id, flavor_ref,
                             block_device_mapping, volumes)
         self.db[id] = server
+        if name.endswith('server_fail'):
+            raise nova_exceptions.ClientException("Fake server create error.")
         server.schedule_status("ACTIVE", 1)
         LOG.info("FAKE_SERVERS_DB : %s" % str(FAKE_SERVERS_DB))
         return server
@@ -339,7 +341,10 @@ class FakeVolumes(object):
         volume = FakeVolume(self, self.context, id, size, display_name,
                             display_description)
         self.db[id] = volume
-        volume.schedule_status("available", 2)
+        if size == 9:
+            volume.schedule_status("error", 2)
+        else:
+            volume.schedule_status("available", 2)
         LOG.info("FAKE_VOLUMES_DB : %s" % FAKE_VOLUMES_DB)
         return volume