From 6df15cebe1b400c0d1c8327ccafb395a3797cb5d Mon Sep 17 00:00:00 2001 From: Sudarshan Acharya Date: Wed, 6 Jun 2012 18:23:11 -0500 Subject: [PATCH] Async operation for instance resize/delete. --- etc/reddwarf/reddwarf-taskmanager.conf.sample | 3 + etc/reddwarf/reddwarf.conf.sample | 10 -- reddwarf/instance/models.py | 111 +----------------- reddwarf/taskmanager/api.py | 13 ++ reddwarf/taskmanager/manager.py | 17 ++- reddwarf/taskmanager/models.py | 95 ++++++++++++++- 6 files changed, 125 insertions(+), 124 deletions(-) diff --git a/etc/reddwarf/reddwarf-taskmanager.conf.sample b/etc/reddwarf/reddwarf-taskmanager.conf.sample index d7b5d4c4e3..49d6619c62 100644 --- a/etc/reddwarf/reddwarf-taskmanager.conf.sample +++ b/etc/reddwarf/reddwarf-taskmanager.conf.sample @@ -47,6 +47,9 @@ reddwarf_auth_url = http://0.0.0.0:5000/v2.0 # Manager impl for the taskmanager taskmanager_manager=reddwarf.taskmanager.manager.TaskManager +# Reddwarf DNS +reddwarf_dns_support = False + # ============ notifer queue kombu connection options ======================== notifier_queue_hostname = localhost diff --git a/etc/reddwarf/reddwarf.conf.sample b/etc/reddwarf/reddwarf.conf.sample index 1595a477bf..34bfb9debd 100644 --- a/etc/reddwarf/reddwarf.conf.sample +++ b/etc/reddwarf/reddwarf.conf.sample @@ -54,16 +54,6 @@ volume_time_out=30 # Reddwarf DNS reddwarf_dns_support = False -#dns_driver=reddwarf.dns.rsdns.driver.RsDnsDriver -#dns_instance_entry_factory=reddwarf.dns.rsdns.driver.RsDnsInstanceEntryFactory -#dns_account_id=0 -#dns_domain_id=0 -#dns_username=username -#dns_passkey=passkey -#dns_domain_name=test.domain-test.com -#dns_management_base_url= -#dns_auth_url= -#dns_ttl=300 # ============ notifer queue kombu connection options ======================== diff --git a/reddwarf/instance/models.py b/reddwarf/instance/models.py index 68973d8107..eca2184a2a 100644 --- a/reddwarf/instance/models.py +++ b/reddwarf/instance/models.py @@ -20,7 +20,6 @@ import eventlet import logging import netaddr -import time from reddwarf import db @@ -36,7 +35,6 @@ from reddwarf.common.remote import create_guest_client from reddwarf.common.remote import create_nova_client from reddwarf.common.remote import create_nova_volume_client from reddwarf.common.utils import poll_until -from reddwarf.guestagent import api as guest_api from reddwarf.instance.tasks import InstanceTask from reddwarf.instance.tasks import InstanceTasks from reddwarf.taskmanager import api as task_api @@ -133,16 +131,6 @@ SERVER_INVALID_ACTION_STATUSES = ["BUILD", "REBOOT", "REBUILD"] VALID_ACTION_STATUSES = ["ACTIVE"] -def ExecuteInstanceMethod(context, id, method_name, *args, **kwargs): - """Loads an instance and executes a method.""" - arg_str = utils.create_method_args_string(*args, **kwargs) - LOG.debug("Loading instance %s to make the following call: %s(%s)." - % (id, method_name, arg_str)) - instance = Instance.load(context, id) - func = getattr(instance, method_name) - func(*args, **kwargs) - - class Instance(object): """Represents an instance. @@ -157,16 +145,6 @@ class Instance(object): self.service_status = service_status self.volumes = volumes - def call_async(self, method, *args, **kwargs): - """Calls a method on this instance in the background and returns. - - This will be a call to some module similar to the guest API, but for - now we just call the real method in eventlet. - - """ - eventlet.spawn(ExecuteInstanceMethod, self.context, self.db_info.id, - method.__name__, *args, **kwargs) - @staticmethod def load(context, id): if context is None: @@ -190,26 +168,10 @@ class Instance(object): % self.id) LOG.debug(_(" ... deleting compute id = %s") % self.server.id) - self._delete_server() LOG.debug(_(" ... setting status to DELETING.")) self.db_info.task_status = InstanceTasks.DELETING self.db_info.save() - #TODO(tim.simpson): Put this in the task manager somehow to shepard - # deletion? - - dns_support = config.Config.get("reddwarf_dns_support", 'False') - LOG.debug(_("reddwarf dns support = %s") % dns_support) - if utils.bool_from_string(dns_support): - dns_client = create_dns_client(self.context) - dns_client.delete_instance_entry(instance_id=self.db_info['id']) - - def _delete_server(self): - try: - self.server.delete() - except nova_exceptions.NotFound, e: - raise rd_exceptions.NotFound(uuid=self.id) - except nova_exceptions.ClientException, e: - raise rd_exceptions.ReddwarfError() + task_api.API(self.context).delete_instance(self.id) @classmethod def _create_volume(cls, context, db_info, volume_size): @@ -428,14 +390,6 @@ class Instance(object): LOG.debug(_(msg) % self.status) raise rd_exceptions.UnprocessableEntity(_(msg) % self.status) - def _refresh_compute_server_info(self): - """Refreshes the compute server field.""" - server, volumes = load_server_with_volumes(self.context, - self.db_info.id, self.db_info.compute_instance_id) - self.server = server - self.volumes = volumes - return server - def resize_flavor(self, new_flavor_id): self.validate_can_perform_resize() LOG.debug("resizing instance %s flavor to %s" @@ -457,67 +411,8 @@ class Instance(object): self.db_info.task_status = InstanceTasks.RESIZING self.db_info.save() LOG.debug("Instance %s set to RESIZING." % self.id) - self.call_async(self.resize_flavor_async, new_flavor_id, - old_flavor_size, new_flavor_size) - - def resize_flavor_async(self, new_flavor_id, old_memory_size, - updated_memory_size): - def resize_status_msg(): - return "instance_id=%s, status=%s, flavor_id=%s, " \ - "dest. flavor id=%s)" % (self.id, self.server.status, \ - str(self.flavor['id']), str(new_flavor_id)) - try: - LOG.debug("Instance %s calling stop_mysql..." % self.id) - guest = create_guest_client(self.context, self.db_info.id) - guest.stop_mysql() - try: - LOG.debug("Instance %s calling Compute resize..." % self.id) - self.server.resize(new_flavor_id) - #TODO(tim.simpson): Figure out some way to message the - # following exceptions: - # nova_exceptions.NotFound (for the flavor) - # nova_exceptions.OverLimit - - self._refresh_compute_server_info() - # Do initial check and confirm the status is appropriate. - if self.server.status != "RESIZE" and \ - self.server.status != "VERIFY_RESIZE": - raise ReddwarfError("Unexpected status after call to " - "resize! : %s" % resize_status_msg()) - - # Wait for the flavor to change. - #TODO(tim.simpson): Bring back our good friend poll_until. - while(self.server.status == "RESIZE"): - LOG.debug("Resizing... currently, %s" % resize_status_msg()) - time.sleep(1) - self._refresh_compute_server_info() - - # Do check to make sure the status and flavor id are correct. - if (str(self.flavor['id']) != str(new_flavor_id) or - self.server.status != "VERIFY_RESIZE"): - raise ReddwarfError("Assertion failed! flavor_id=%s " - "and not %s" - % (self.server.status, str(self.flavor['id']))) - - # Confirm the resize with Nova. - LOG.debug("Instance %s calling Compute confirm resize..." - % self.id) - self.server.confirm_resize() - except Exception as ex: - updated_memory_size = old_memory_size - LOG.error("Error during resize compute! Aborting action.") - LOG.error(ex) - raise - finally: - # Tell the guest to restart MySQL with the new RAM size. - # This is in the finally because we have to call this, or - # else MySQL could stay turned off on an otherwise usable - # instance. - LOG.debug("Instance %s starting mysql..." % self.id) - guest.start_mysql_with_conf_changes(updated_memory_size) - finally: - self.db_info.task_status = InstanceTasks.NONE - self.db_info.save() + task_api.API(self.context).resize_flavor(self.id, new_flavor_id, + old_flavor_size, new_flavor_size) def resize_volume(self, new_size): LOG.info("Resizing volume of instance %s..." % self.id) diff --git a/reddwarf/taskmanager/api.py b/reddwarf/taskmanager/api.py index 99ab017b07..b6f0b3a369 100644 --- a/reddwarf/taskmanager/api.py +++ b/reddwarf/taskmanager/api.py @@ -60,3 +60,16 @@ class API(object): LOG.debug("Making async call to resize volume for instance: %s" % instance_id) self._cast("resize_volume", new_size=new_size, instance_id=instance_id) + + def resize_flavor(self, instance_id, new_flavor_id, old_flavor_size, + new_flavor_size): + LOG.debug("Making async call to resize flavor for instance: %s" % + instance_id) + self._cast("resize_flavor", instance_id=instance_id, + new_flavor_id=new_flavor_id, + old_flavor_size=old_flavor_size, + new_flavor_size=new_flavor_size) + + def delete_instance(self, instance_id): + LOG.debug("Making async call to delete instance: %s" % instance_id) + self._cast("delete_instance", instance_id=instance_id) diff --git a/reddwarf/taskmanager/manager.py b/reddwarf/taskmanager/manager.py index ebcfb8a780..ae6c2401f1 100644 --- a/reddwarf/taskmanager/manager.py +++ b/reddwarf/taskmanager/manager.py @@ -16,11 +16,11 @@ # under the License. import logging +import traceback import weakref from eventlet import greenthread -from reddwarf.common import excutils from reddwarf.common import service from reddwarf.taskmanager import models @@ -48,10 +48,23 @@ class TaskManager(service.Manager): func = getattr(self, method) func(context, *args, **kwargs) except Exception as e: - excutils.save_and_reraise_exception() + LOG.error("Got an error running %s!" % method) + LOG.error(traceback.format_exc()) finally: del self.tasks[greenthread.getcurrent()] def resize_volume(self, context, instance_id, new_size): instance_tasks = models.InstanceTasks.load(context, instance_id) instance_tasks.resize_volume(new_size) + + def resize_flavor(self, context, instance_id, new_flavor_id, + old_flavor_size, new_flavor_size): + instance_tasks = models.InstanceTasks.load(context, instance_id) + instance_tasks.resize_flavor(new_flavor_id, old_flavor_size, + new_flavor_size) + + def delete_instance(self, context, instance_id): + instance_tasks = models.InstanceTasks.load(context, instance_id) + instance_tasks.delete_instance() + + diff --git a/reddwarf/taskmanager/models.py b/reddwarf/taskmanager/models.py index 741c95425c..5814833c85 100644 --- a/reddwarf/taskmanager/models.py +++ b/reddwarf/taskmanager/models.py @@ -16,10 +16,13 @@ import logging from eventlet import greenthread +from novaclient import exceptions as nova_exceptions from reddwarf.common import config -from reddwarf.common import exception from reddwarf.common import remote from reddwarf.common import utils +from reddwarf.common.exception import PollTimeOut +from reddwarf.common.exception import ReddwarfError +from reddwarf.common.remote import create_dns_client from reddwarf.instance import models as inst_models @@ -61,8 +64,8 @@ class InstanceTasks: raise TypeError("Argument id not defined.") try: db_info = inst_models.DBInstance.find_by(id=id) - except exception.NotFound: - raise exception.NotFound(uuid=id) + except NotFound: + raise NotFound(uuid=id) server, volumes = inst_models.load_server_with_volumes(context, db_info.id, db_info.compute_instance_id) @@ -73,6 +76,25 @@ class InstanceTasks: nova_client=nova_client, volume_client=volume_client, guest=guest) + def delete_instance(self): + try: + self.server.delete() + except Exception as ex: + LOG.error("Error during delete compute server %s " + % self.server.id) + LOG.error(ex) + + try: + dns_support = config.Config.get("reddwarf_dns_support", 'False') + LOG.debug(_("reddwarf dns support = %s") % dns_support) + if utils.bool_from_string(dns_support): + dns_api = create_dns_client(self.context) + dns_api.delete_instance_entry(instance_id=self.db_info.id) + except Exception as ex: + LOG.error("Error during dns entry for instance %s " + % self.db_info.id ) + LOG.error(ex) + def resize_volume(self, new_size): LOG.debug("%s: Resizing volume for instance: %s to %r GB" % (greenthread.getcurrent(), self.server.id, new_size)) @@ -86,7 +108,7 @@ class InstanceTasks: self.nova_client.volumes.rescan_server_volume(self.server, self.volume_id) self.guest.resize_fs(self.volume_mountpoint) - except exception.PollTimeOut as pto: + except PollTimeOut as pto: LOG.error("Timeout trying to rescan or resize the attached volume " "filesystem for volume: %s" % self.volume_id) except Exception as e: @@ -96,3 +118,68 @@ class InstanceTasks: finally: self.db_info.task_status = inst_models.InstanceTasks.NONE self.db_info.save() + + def resize_flavor(self, new_flavor_id, old_memory_size, + new_memory_size): + def resize_status_msg(): + return "instance_id=%s, status=%s, flavor_id=%s, "\ + "dest. flavor id=%s)" % (self.db_info.id, + self.server.status, + str(self.flavor['id']), + str(new_flavor_id)) + + try: + LOG.debug("Instance %s calling stop_mysql..." % self.db_info.id) + self.guest.stop_mysql() + try: + LOG.debug("Instance %s calling Compute resize..." + % self.db_info.id) + self.server.resize(new_flavor_id) + + # Do initial check and confirm the status is appropriate. + self._refresh_compute_server_info() + if self.server.status != "RESIZE" and\ + self.server.status != "VERIFY_RESIZE": + raise ReddwarfError("Unexpected status after " \ + "call to resize! : %s" % resize_status_msg()) + + # Wait for the flavor to change. + utils.poll_until( + lambda: self.nova_client.servers.get(self.server.id), + lambda server: server.status != 'RESIZE', + sleep_time=2, + time_out=60 * 2) + + # Do check to make sure the status and flavor id are correct. + if (str(self.server.flavor['id']) != str(new_flavor_id) or + self.server.status != "VERIFY_RESIZE"): + raise ReddwarfError("Assertion failed! flavor_id=%s " + "and not %s" + % (self.server.status, str(self.server.flavor['id']))) + + # Confirm the resize with Nova. + LOG.debug("Instance %s calling Compute confirm resize..." + % self.db_info.id) + self.server.confirm_resize() + except PollTimeOut as pto: + LOG.error("Timeout trying to resize the flavor for instance " + " %s" % self.db_info.id) + except Exception as ex: + new_memory_size = old_memory_size + LOG.error("Error during resize compute! Aborting action.") + LOG.error(ex) + finally: + # Tell the guest to restart MySQL with the new RAM size. + # This is in the finally because we have to call this, or + # else MySQL could stay turned off on an otherwise usable + # instance. + LOG.debug("Instance %s starting mysql..." % self.db_info.id) + self.guest.start_mysql_with_conf_changes(new_memory_size) + finally: + self.db_info.task_status = inst_models.InstanceTasks.NONE + self.db_info.save() + + def _refresh_compute_server_info(self): + """Refreshes the compute server field.""" + server = self.nova_client.servers.get(self.server.id) + self.server = server