openstack/trove
Code Issues Proposed changes

Async operation for instance resize/delete.

Browse Source
This commit is contained in:
Sudarshan Acharya 2012-06-06 18:23:11 -05:00
parent b701c89f8d
commit 6df15cebe1
6 changed files with 125 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
etc/reddwarf/reddwarf-taskmanager.conf.sample
View File

@ -47,6 +47,9 @@ reddwarf_auth_url = http://0.0.0.0:5000/v2.0
# Manager impl for the taskmanager
taskmanager_manager=reddwarf.taskmanager.manager.TaskManager
# Reddwarf DNS
reddwarf_dns_support = False
# ============ notifer queue kombu connection options ========================
notifier_queue_hostname = localhost

10
etc/reddwarf/reddwarf.conf.sample
View File

@ -54,16 +54,6 @@ volume_time_out=30
# Reddwarf DNS
reddwarf_dns_support = False
#dns_driver=reddwarf.dns.rsdns.driver.RsDnsDriver
#dns_instance_entry_factory=reddwarf.dns.rsdns.driver.RsDnsInstanceEntryFactory
#dns_account_id=0
#dns_domain_id=0
#dns_username=username
#dns_passkey=passkey
#dns_domain_name=test.domain-test.com
#dns_management_base_url=
#dns_auth_url=
#dns_ttl=300
# ============ notifer queue kombu connection options ========================

111
reddwarf/instance/models.py
View File

@ -20,7 +20,6 @@
import eventlet
import logging
import netaddr
import time
from reddwarf import db
@ -36,7 +35,6 @@ from reddwarf.common.remote import create_guest_client
from reddwarf.common.remote import create_nova_client
from reddwarf.common.remote import create_nova_volume_client
from reddwarf.common.utils import poll_until
from reddwarf.guestagent import api as guest_api
from reddwarf.instance.tasks import InstanceTask
from reddwarf.instance.tasks import InstanceTasks
from reddwarf.taskmanager import api as task_api
@ -133,16 +131,6 @@ SERVER_INVALID_ACTION_STATUSES = ["BUILD", "REBOOT", "REBUILD"]
VALID_ACTION_STATUSES = ["ACTIVE"]
def ExecuteInstanceMethod(context, id, method_name, *args, **kwargs):
"""Loads an instance and executes a method."""
arg_str = utils.create_method_args_string(*args, **kwargs)
LOG.debug("Loading instance %s to make the following call: %s(%s)."
% (id, method_name, arg_str))
instance = Instance.load(context, id)
func = getattr(instance, method_name)
func(*args, **kwargs)
class Instance(object):
"""Represents an instance.
@ -157,16 +145,6 @@ class Instance(object):
self.service_status = service_status
self.volumes = volumes
def call_async(self, method, *args, **kwargs):
"""Calls a method on this instance in the background and returns.
This will be a call to some module similar to the guest API, but for
now we just call the real method in eventlet.
"""
eventlet.spawn(ExecuteInstanceMethod, self.context, self.db_info.id,
method.__name__, *args, **kwargs)
@staticmethod
def load(context, id):
if context is None:
@ -190,26 +168,10 @@ class Instance(object):
% self.id)
LOG.debug(_(" ... deleting compute id = %s") %
self.server.id)
self._delete_server()
LOG.debug(_(" ... setting status to DELETING."))
self.db_info.task_status = InstanceTasks.DELETING
self.db_info.save()
#TODO(tim.simpson): Put this in the task manager somehow to shepard
# deletion?
dns_support = config.Config.get("reddwarf_dns_support", 'False')
LOG.debug(_("reddwarf dns support = %s") % dns_support)
if utils.bool_from_string(dns_support):
dns_client = create_dns_client(self.context)
dns_client.delete_instance_entry(instance_id=self.db_info['id'])
def _delete_server(self):
try:
self.server.delete()
except nova_exceptions.NotFound, e:
raise rd_exceptions.NotFound(uuid=self.id)
except nova_exceptions.ClientException, e:
raise rd_exceptions.ReddwarfError()
task_api.API(self.context).delete_instance(self.id)
@classmethod
def _create_volume(cls, context, db_info, volume_size):
@ -428,14 +390,6 @@ class Instance(object):
LOG.debug(_(msg) % self.status)
raise rd_exceptions.UnprocessableEntity(_(msg) % self.status)
def _refresh_compute_server_info(self):
"""Refreshes the compute server field."""
server, volumes = load_server_with_volumes(self.context,
self.db_info.id, self.db_info.compute_instance_id)
self.server = server
self.volumes = volumes
return server
def resize_flavor(self, new_flavor_id):
self.validate_can_perform_resize()
LOG.debug("resizing instance %s flavor to %s"
@ -457,67 +411,8 @@ class Instance(object):
self.db_info.task_status = InstanceTasks.RESIZING
self.db_info.save()
LOG.debug("Instance %s set to RESIZING." % self.id)
self.call_async(self.resize_flavor_async, new_flavor_id,
old_flavor_size, new_flavor_size)
def resize_flavor_async(self, new_flavor_id, old_memory_size,
updated_memory_size):
def resize_status_msg():
return "instance_id=%s, status=%s, flavor_id=%s, " \
"dest. flavor id=%s)" % (self.id, self.server.status, \
str(self.flavor['id']), str(new_flavor_id))
try:
LOG.debug("Instance %s calling stop_mysql..." % self.id)
guest = create_guest_client(self.context, self.db_info.id)
guest.stop_mysql()
try:
LOG.debug("Instance %s calling Compute resize..." % self.id)
self.server.resize(new_flavor_id)
#TODO(tim.simpson): Figure out some way to message the
# following exceptions:
# nova_exceptions.NotFound (for the flavor)
# nova_exceptions.OverLimit
self._refresh_compute_server_info()
# Do initial check and confirm the status is appropriate.
if self.server.status != "RESIZE" and \
self.server.status != "VERIFY_RESIZE":
raise ReddwarfError("Unexpected status after call to "
"resize! : %s" % resize_status_msg())
# Wait for the flavor to change.
#TODO(tim.simpson): Bring back our good friend poll_until.
while(self.server.status == "RESIZE"):
LOG.debug("Resizing... currently, %s" % resize_status_msg())
time.sleep(1)
self._refresh_compute_server_info()
# Do check to make sure the status and flavor id are correct.
if (str(self.flavor['id']) != str(new_flavor_id) or
self.server.status != "VERIFY_RESIZE"):
raise ReddwarfError("Assertion failed! flavor_id=%s "
"and not %s"
% (self.server.status, str(self.flavor['id'])))
# Confirm the resize with Nova.
LOG.debug("Instance %s calling Compute confirm resize..."
% self.id)
self.server.confirm_resize()
except Exception as ex:
updated_memory_size = old_memory_size
LOG.error("Error during resize compute! Aborting action.")
LOG.error(ex)
raise
finally:
# Tell the guest to restart MySQL with the new RAM size.
# This is in the finally because we have to call this, or
# else MySQL could stay turned off on an otherwise usable
# instance.
LOG.debug("Instance %s starting mysql..." % self.id)
guest.start_mysql_with_conf_changes(updated_memory_size)
finally:
self.db_info.task_status = InstanceTasks.NONE
self.db_info.save()
task_api.API(self.context).resize_flavor(self.id, new_flavor_id,
old_flavor_size, new_flavor_size)
def resize_volume(self, new_size):
LOG.info("Resizing volume of instance %s..." % self.id)

13
reddwarf/taskmanager/api.py
View File

@ -60,3 +60,16 @@ class API(object):
LOG.debug("Making async call to resize volume for instance: %s"
% instance_id)
self._cast("resize_volume", new_size=new_size, instance_id=instance_id)
def resize_flavor(self, instance_id, new_flavor_id, old_flavor_size,
new_flavor_size):
LOG.debug("Making async call to resize flavor for instance: %s" %
instance_id)
self._cast("resize_flavor", instance_id=instance_id,
new_flavor_id=new_flavor_id,
old_flavor_size=old_flavor_size,
new_flavor_size=new_flavor_size)
def delete_instance(self, instance_id):
LOG.debug("Making async call to delete instance: %s" % instance_id)
self._cast("delete_instance", instance_id=instance_id)

17
reddwarf/taskmanager/manager.py
View File

@ -16,11 +16,11 @@
# under the License.
import logging
import traceback
import weakref
from eventlet import greenthread
from reddwarf.common import excutils
from reddwarf.common import service
from reddwarf.taskmanager import models
@ -48,10 +48,23 @@ class TaskManager(service.Manager):
func = getattr(self, method)
func(context, *args, **kwargs)
except Exception as e:
excutils.save_and_reraise_exception()
LOG.error("Got an error running %s!" % method)
LOG.error(traceback.format_exc())
finally:
del self.tasks[greenthread.getcurrent()]
def resize_volume(self, context, instance_id, new_size):
instance_tasks = models.InstanceTasks.load(context, instance_id)
instance_tasks.resize_volume(new_size)
def resize_flavor(self, context, instance_id, new_flavor_id,
old_flavor_size, new_flavor_size):
instance_tasks = models.InstanceTasks.load(context, instance_id)
instance_tasks.resize_flavor(new_flavor_id, old_flavor_size,
new_flavor_size)
def delete_instance(self, context, instance_id):
instance_tasks = models.InstanceTasks.load(context, instance_id)
instance_tasks.delete_instance()

95
reddwarf/taskmanager/models.py
View File

@ -16,10 +16,13 @@ import logging
from eventlet import greenthread
from novaclient import exceptions as nova_exceptions
from reddwarf.common import config
from reddwarf.common import exception
from reddwarf.common import remote
from reddwarf.common import utils
from reddwarf.common.exception import PollTimeOut
from reddwarf.common.exception import ReddwarfError
from reddwarf.common.remote import create_dns_client
from reddwarf.instance import models as inst_models
@ -61,8 +64,8 @@ class InstanceTasks:
raise TypeError("Argument id not defined.")
try:
db_info = inst_models.DBInstance.find_by(id=id)
except exception.NotFound:
raise exception.NotFound(uuid=id)
except NotFound:
raise NotFound(uuid=id)
server, volumes = inst_models.load_server_with_volumes(context,
db_info.id,
db_info.compute_instance_id)
@ -73,6 +76,25 @@ class InstanceTasks:
nova_client=nova_client,
volume_client=volume_client, guest=guest)
def delete_instance(self):
try:
self.server.delete()
except Exception as ex:
LOG.error("Error during delete compute server %s "
% self.server.id)
LOG.error(ex)
try:
dns_support = config.Config.get("reddwarf_dns_support", 'False')
LOG.debug(_("reddwarf dns support = %s") % dns_support)
if utils.bool_from_string(dns_support):
dns_api = create_dns_client(self.context)
dns_api.delete_instance_entry(instance_id=self.db_info.id)
except Exception as ex:
LOG.error("Error during dns entry for instance %s "
% self.db_info.id )
LOG.error(ex)
def resize_volume(self, new_size):
LOG.debug("%s: Resizing volume for instance: %s to %r GB"
% (greenthread.getcurrent(), self.server.id, new_size))
@ -86,7 +108,7 @@ class InstanceTasks:
self.nova_client.volumes.rescan_server_volume(self.server,
self.volume_id)
self.guest.resize_fs(self.volume_mountpoint)
except exception.PollTimeOut as pto:
except PollTimeOut as pto:
LOG.error("Timeout trying to rescan or resize the attached volume "
"filesystem for volume: %s" % self.volume_id)
except Exception as e:
@ -96,3 +118,68 @@ class InstanceTasks:
finally:
self.db_info.task_status = inst_models.InstanceTasks.NONE
self.db_info.save()
def resize_flavor(self, new_flavor_id, old_memory_size,
new_memory_size):
def resize_status_msg():
return "instance_id=%s, status=%s, flavor_id=%s, "\
"dest. flavor id=%s)" % (self.db_info.id,
self.server.status,
str(self.flavor['id']),
str(new_flavor_id))
try:
LOG.debug("Instance %s calling stop_mysql..." % self.db_info.id)
self.guest.stop_mysql()
try:
LOG.debug("Instance %s calling Compute resize..."
% self.db_info.id)
self.server.resize(new_flavor_id)
# Do initial check and confirm the status is appropriate.
self._refresh_compute_server_info()
if self.server.status != "RESIZE" and\
self.server.status != "VERIFY_RESIZE":
raise ReddwarfError("Unexpected status after " \
"call to resize! : %s" % resize_status_msg())
# Wait for the flavor to change.
utils.poll_until(
lambda: self.nova_client.servers.get(self.server.id),
lambda server: server.status != 'RESIZE',
sleep_time=2,
time_out=60 * 2)
# Do check to make sure the status and flavor id are correct.
if (str(self.server.flavor['id']) != str(new_flavor_id) or
self.server.status != "VERIFY_RESIZE"):
raise ReddwarfError("Assertion failed! flavor_id=%s "
"and not %s"
% (self.server.status, str(self.server.flavor['id'])))
# Confirm the resize with Nova.
LOG.debug("Instance %s calling Compute confirm resize..."
% self.db_info.id)
self.server.confirm_resize()
except PollTimeOut as pto:
LOG.error("Timeout trying to resize the flavor for instance "
" %s" % self.db_info.id)
except Exception as ex:
new_memory_size = old_memory_size
LOG.error("Error during resize compute! Aborting action.")
LOG.error(ex)
finally:
# Tell the guest to restart MySQL with the new RAM size.
# This is in the finally because we have to call this, or
# else MySQL could stay turned off on an otherwise usable
# instance.
LOG.debug("Instance %s starting mysql..." % self.db_info.id)
self.guest.start_mysql_with_conf_changes(new_memory_size)
finally:
self.db_info.task_status = inst_models.InstanceTasks.NONE
self.db_info.save()
def _refresh_compute_server_info(self):
"""Refreshes the compute server field."""
server = self.nova_client.servers.get(self.server.id)
self.server = server