diff --git a/reddwarf/common/exception.py b/reddwarf/common/exception.py index c0976e5964..c3885f282d 100644 --- a/reddwarf/common/exception.py +++ b/reddwarf/common/exception.py @@ -58,6 +58,11 @@ class NotFound(ReddwarfError): message = _("Resource %(uuid)s cannot be found") +class FlavorNotFound(ReddwarfError): + + message = _("Resource %(uuid)s cannot be found") + + class ComputeInstanceNotFound(NotFound): internal_message = _("Cannot find compute instance %(server_id)s for " @@ -66,10 +71,16 @@ class ComputeInstanceNotFound(NotFound): message = _("Resource %(instance_id)s can not be retrieved.") +class OverLimit(ReddwarfError): + + internal_message = _("The server rejected the request due to its size or " + "rate.") + + class GuestError(ReddwarfError): message = _("An error occurred communicating with the guest: " - "%(original_message).") + "%(original_message)s.") class BadRequest(ReddwarfError): @@ -88,7 +99,17 @@ class UnprocessableEntity(ReddwarfError): message = _("Unable to process the contained request") +class CannotResizeToSameSize(ReddwarfError): + + message = _("When resizing, instances must change size!") + + class VolumeAttachmentsNotFound(NotFound): message = _("Cannot find the volumes attached to compute " "instance %(server_id)") + + +class VolumeCreationFailure(ReddwarfError): + + message = _("Failed to create a volume in Nova.") diff --git a/reddwarf/common/utils.py b/reddwarf/common/utils.py index c007aafc3d..8b50ae19ed 100644 --- a/reddwarf/common/utils.py +++ b/reddwarf/common/utils.py @@ -42,6 +42,19 @@ execute = openstack_utils.execute isotime = openstack_utils.isotime +def create_method_args_string(*args, **kwargs): + """Returns a string representation of args and keyword args. + + I.e. for args=1,2,3 and kwargs={'a':4, 'b':5} you'd get: "1,2,3,a=4,b=5" + """ + # While %s turns a var into a string but in some rare cases explicit + # repr() is less likely to raise an exception. + arg_strs = [repr(arg) for arg in args] + arg_strs += ['%s=%s' % (repr(key), repr(value)) + for (key, value) in kwargs.items()] + return ', '.join(arg_strs) + + def stringify_keys(dictionary): if dictionary is None: return None diff --git a/reddwarf/db/sqlalchemy/migrate_repo/versions/001_base_schema.py b/reddwarf/db/sqlalchemy/migrate_repo/versions/001_base_schema.py index 91d5f9d353..087ad91908 100644 --- a/reddwarf/db/sqlalchemy/migrate_repo/versions/001_base_schema.py +++ b/reddwarf/db/sqlalchemy/migrate_repo/versions/001_base_schema.py @@ -40,7 +40,8 @@ instances = Table('instances', meta, Column('compute_instance_id', String(36)), Column('task_id', Integer()), Column('task_description', String(32)), - Column('task_start_time', DateTime())) + Column('task_start_time', DateTime()), + Column('volume_id', String(36))) def upgrade(migrate_engine): diff --git a/reddwarf/flavor/views.py b/reddwarf/flavor/views.py index 82508e0ed9..5d2c530eee 100644 --- a/reddwarf/flavor/views.py +++ b/reddwarf/flavor/views.py @@ -40,7 +40,7 @@ class FlavorView(object): detailed = '/detail' splitpath.pop(-1) flavorid = self.flavor.id - if splitpath[-1] == flavorid: + if str(splitpath[-1]) == str(flavorid): splitpath.pop(-1) href_template = "%(scheme)s://%(endpoint)s%(path)s/%(flavorid)s" for link in self.flavor.links: diff --git a/reddwarf/guestagent/dbaas.py b/reddwarf/guestagent/dbaas.py index 7a5bca84ab..d118b42504 100644 --- a/reddwarf/guestagent/dbaas.py +++ b/reddwarf/guestagent/dbaas.py @@ -46,7 +46,7 @@ from reddwarf.common.exception import ProcessExecutionError from reddwarf.common import config from reddwarf.common import utils from reddwarf.guestagent.db import models -from reddwarf.guestagent.volume import VolumeHelper +from reddwarf.guestagent.volume import VolumeDevice from reddwarf.instance import models as rd_models @@ -77,7 +77,8 @@ def get_auth_password(): "/password\\t=/{print $3}", "/etc/mysql/my.cnf") if err: LOG.err(err) - raise RuntimeError("Problem reading my.cnf! : %s" % err) + raise RuntimeError("Problem reading my.cnf! : %s" % err) + return pwd.strip() def get_engine(): @@ -89,15 +90,11 @@ def get_engine(): if ENGINE: return ENGINE #ENGINE = create_engine(name_or_url=url) - pwd, err = utils.execute_with_timeout("sudo", "awk", - "/password\\t=/{print $3}", "/etc/mysql/my.cnf") - if not err: - ENGINE = create_engine("mysql://%s:%s@localhost:3306" % - (ADMIN_USER_NAME, pwd.strip()), - pool_recycle=7200, echo=True, - listeners=[KeepAliveConnection()]) - else: - LOG.error(err) + pwd = get_auth_password() + ENGINE = create_engine("mysql://%s:%s@localhost:3306" % + (ADMIN_USER_NAME, pwd.strip()), + pool_recycle=7200, echo=True, + listeners=[KeepAliveConnection()]) return ENGINE @@ -328,9 +325,10 @@ class MySqlAdmin(object): user.deserialize(item) # TODO(cp16net):Should users be allowed to create users # 'os_admin' or 'debian-sys-maint' - t = text("""CREATE USER `%s`@:host IDENTIFIED BY '%s';""" - % (user.name, user.password)) - client.execute(t, host=host) + t = text("""GRANT USAGE ON *.* TO '%s'@\"%s\" + IDENTIFIED BY '%s';""" + % (user.name, host, user.password)) + client.execute(t) for database in user.databases: mydb = models.MySQLDatabase() mydb.deserialize(database) @@ -505,17 +503,16 @@ class DBaaSAgent(object): app = MySqlApp(self.status) restart_mysql = False if device_path: - VolumeHelper.format(device_path) + device = VolumeDevice(device_path) + device.format() if app.is_installed(pkg): #stop and do not update database app.stop_mysql() restart_mysql = True #rsync exiting data - VolumeHelper.migrate_data(device_path, MYSQL_BASE_DIR) + device.migrate_data(MYSQL_BASE_DIR) #mount the volume - VolumeHelper.mount(device_path, mount_point) - #TODO(cp16net) need to update the fstab here so that on a - # restart the volume will be mounted automatically again + device.mount(mount_point) LOG.debug(_("Mounted the volume.")) #check mysql was installed and stopped if restart_mysql: @@ -531,6 +528,15 @@ class DBaaSAgent(object): app = MySqlApp(self.status) app.restart() + def start_mysql_with_conf_changes(self, updated_memory_size): + app = MySqlApp(self.status) + pkg = self # Python cast. + app.start_mysql_with_conf_changes(pkg, updated_memory_size) + + def stop_mysql(self): + app = MySqlApp(self.status) + app.stop_mysql() + def update_status(self): """Update the status of the MySQL service""" MySqlAppStatus.get().update() @@ -646,6 +652,14 @@ class MySqlApp(object): AND Host!='localhost';""") client.execute(t) + def restart_with_sync(self, migration_function): + """Restarts MySQL, doing some action in-between. + + Does not update the database.""" + self._internal_stop_mysql() + migration_function() + self.start_mysql() + def restart(self): try: self.status.begin_mysql_restart() @@ -654,16 +668,6 @@ class MySqlApp(object): finally: self.status.end_install_or_restart() - # def _restart_mysql_and_wipe_ib_logfiles(self): - # """Stops MySQL and restarts it, wiping the ib_logfiles in-between. - - # This should never be done unless the innodb_log_file_size changes. - # """ - # LOG.info("Restarting mysql...") - # self._internal_stop_mysql() - # self._wipe_ib_logfiles() - # self._start_mysql() - def _replace_mycnf_with_template(self, template_path, original_path): if os.path.isfile(template_path): utils.execute_with_timeout("sudo", "mv", original_path, @@ -685,11 +689,25 @@ class MySqlApp(object): tmp_file.close() def wipe_ib_logfiles(self): + """Destroys the iblogfiles. + + If for some reason the selected log size in the conf changes from the + current size of the files MySQL will fail to start, so we delete the + files to be safe. + """ LOG.info(_("Wiping ib_logfiles...")) - utils.execute_with_timeout("sudo", "rm", "%s/ib_logfile0" - % MYSQL_BASE_DIR) - utils.execute_with_timeout("sudo", "rm", "%s/ib_logfile1" - % MYSQL_BASE_DIR) + for index in range(2): + try: + utils.execute_with_timeout("sudo", "rm", "%s/ib_logfile%d" + % (MYSQL_BASE_DIR, index)) + except ProcessExecutionError as pe: + # On restarts, sometimes these are wiped. So it can be a race + # to have MySQL start up before it's restarted and these have + # to be deleted. That's why its ok if they aren't found. + LOG.error("Could not delete logfile!") + LOG.error(pe) + if "No such file or directory" not in str(pe): + raise def _write_mycnf(self, pkg, update_memory_mb, admin_password): """ @@ -754,7 +772,7 @@ class MySqlApp(object): self.status.end_install_or_restart() raise RuntimeError("Could not start MySQL!") - def start_mysl_with_conf_changes(self, pkg, updated_memory_mb): + def start_mysql_with_conf_changes(self, pkg, updated_memory_mb): LOG.info(_("Starting mysql with conf changes...")) if self.status.is_mysql_running: LOG.error(_("Cannot execute start_mysql_with_conf_changes because " diff --git a/reddwarf/guestagent/volume.py b/reddwarf/guestagent/volume.py index 3a126d0b77..f29f5f7d51 100644 --- a/reddwarf/guestagent/volume.py +++ b/reddwarf/guestagent/volume.py @@ -30,26 +30,24 @@ LOG = logging.getLogger(__name__) CONFIG = config.Config -class VolumeHelper(object): +class VolumeDevice(object): - @staticmethod - def _has_volume_device(device_path): - return not device_path is None + def __init__(self, device_path): + self.device_path = device_path - @staticmethod - def migrate_data(device_path, mysql_base): + def migrate_data(self, mysql_base): """ Synchronize the data from the mysql directory to the new volume """ + # Use sudo to have access to this spot. utils.execute("sudo", "mkdir", "-p", TMP_MOUNT_POINT) - VolumeHelper.mount(device_path, TMP_MOUNT_POINT) + self._tmp_mount(TMP_MOUNT_POINT) if not mysql_base[-1] == '/': mysql_base = "%s/" % mysql_base utils.execute("sudo", "rsync", "--safe-links", "--perms", "--recursive", "--owner", "--group", "--xattrs", "--sparse", mysql_base, TMP_MOUNT_POINT) - VolumeHelper.unmount(device_path) + self.unmount() - @staticmethod - def _check_device_exists(device_path): + def _check_device_exists(self): """Check that the device path exists. Verify that the device path has actually been created and can report @@ -58,72 +56,100 @@ class VolumeHelper(object): """ try: num_tries = CONFIG.get('num_tries', 3) - utils.execute('sudo', 'blockdev', '--getsize64', device_path, + utils.execute('sudo', 'blockdev', '--getsize64', self.device_path, attempts=num_tries) except ProcessExecutionError: - raise GuestError("InvalidDevicePath(path=%s)" % device_path) + raise GuestError("InvalidDevicePath(path=%s)" % self.device_path) - @staticmethod - def _check_format(device_path): + def _check_format(self): """Checks that an unmounted volume is formatted.""" - child = pexpect.spawn("sudo dumpe2fs %s" % device_path) + child = pexpect.spawn("sudo dumpe2fs %s" % self.device_path) try: i = child.expect(['has_journal', 'Wrong magic number']) if i == 0: return volume_fstype = CONFIG.get('volume_fstype', 'ext3') raise IOError('Device path at %s did not seem to be %s.' % - (device_path, volume_fstype)) + (self.device_path, volume_fstype)) except pexpect.EOF: raise IOError("Volume was not formatted.") child.expect(pexpect.EOF) - @staticmethod - def _format(device_path): + def _format(self): """Calls mkfs to format the device at device_path.""" volume_fstype = CONFIG.get('volume_fstype', 'ext3') format_options = CONFIG.get('format_options', '-m 5') cmd = "sudo mkfs -t %s %s %s" % (volume_fstype, - format_options, device_path) + format_options, self.device_path) volume_format_timeout = CONFIG.get('volume_format_timeout', 120) child = pexpect.spawn(cmd, timeout=volume_format_timeout) # child.expect("(y,n)") # child.sendline('y') child.expect(pexpect.EOF) - @staticmethod - def format(device_path): + def format(self): """Formats the device at device_path and checks the filesystem.""" - VolumeHelper._check_device_exists(device_path) - VolumeHelper._format(device_path) - VolumeHelper._check_format(device_path) + self._check_device_exists() + self._format() + self._check_format() - @staticmethod - def mount(device_path, mount_point): - if not os.path.exists(mount_point): - os.makedirs(mount_point) - volume_fstype = CONFIG.get('volume_fstype', 'ext3') - mount_options = CONFIG.get('mount_options', 'noatime') - cmd = "sudo mount -t %s -o %s %s %s" % (volume_fstype, - mount_options, - device_path, mount_point) - child = pexpect.spawn(cmd) - child.expect(pexpect.EOF) + def mount(self, mount_point): + """Mounts, and writes to fstab.""" + mount_point = VolumeMountPoint(self.device_path, mount_point) + mount_point.mount() + mount_point.write_to_fstab() - @staticmethod - def unmount(mount_point): - if os.path.exists(mount_point): - cmd = "sudo umount %s" % mount_point - child = pexpect.spawn(cmd) - child.expect(pexpect.EOF) - - @staticmethod - def resize_fs(device_path): + #TODO(tim.simpson): Are we using this? + def resize_fs(self): """Resize the filesystem on the specified device""" - VolumeHelper._check_device_exists(device_path) + self._check_device_exists() try: - utils.execute("sudo", "resize2fs", device_path) + utils.execute("sudo", "resize2fs", self.device_path) except ProcessExecutionError as err: LOG.error(err) raise GuestError("Error resizing the filesystem: %s" - % device_path) + % self.device_path) + + def _tmp_mount(self, mount_point): + """Mounts, but doesn't save to fstab.""" + mount_point = VolumeMountPoint(self.device_path, mount_point) + mount_point.mount() # Don't save to fstab. + + def unmount(self): + if os.path.exists(self.device_path): + cmd = "sudo umount %s" % self.device_path + child = pexpect.spawn(cmd) + child.expect(pexpect.EOF) + + +class VolumeMountPoint(object): + + def __init__(self, device_path, mount_point): + self.device_path = device_path + self.mount_point = mount_point + self.volume_fstype = CONFIG.get('volume_fstype', 'ext3') + self.mount_options = CONFIG.get('mount_options', 'defaults,noatime') + + def mount(self): + if not os.path.exists(self.mount_point): + os.makedirs(self.mount_point) + LOG.debug("Adding volume. Device path:%s, mount_point:%s, " + "volume_type:%s, mount options:%s" % + (self.device_path, self.mount_point, self.volume_fstype, + self.mount_options)) + cmd = "sudo mount -t %s -o %s %s %s" % (self.volume_fstype, + self.mount_options, self.device_path, self.mount_point) + child = pexpect.spawn(cmd) + child.expect(pexpect.EOF) + + def write_to_fstab(self): + fstab_line = "%s\t%s\t%s\t%s\t0\t0" % (self.device_path, + self.mount_point, self.volume_fstype, self.mount_options) + LOG.debug("Writing new line to fstab:%s" % fstab_line) + utils.execute("sudo", "cp", "/etc/fstab", "/etc/fstab.orig") + utils.execute("sudo", "cp", "/etc/fstab", "/tmp/newfstab") + utils.execute("sudo", "chmod", "666", "/tmp/newfstab") + with open("/tmp/newfstab", 'a') as new_fstab: + new_fstab.write("\n" + fstab_line) + utils.execute("sudo", "chmod", "640", "/tmp/newfstab") + utils.execute("sudo", "mv", "/tmp/newfstab", "/etc/fstab") diff --git a/reddwarf/instance/models.py b/reddwarf/instance/models.py index fe3e343dd4..32a7c2d280 100644 --- a/reddwarf/instance/models.py +++ b/reddwarf/instance/models.py @@ -17,8 +17,10 @@ """Model classes that form the core of instances functionality.""" +import eventlet import logging import netaddr +import time from reddwarf import db @@ -41,7 +43,7 @@ CONFIG = config.Config LOG = logging.getLogger(__name__) -def load_server(context, instance_id, server_id): +def load_server_with_volumes(context, instance_id, server_id): """Loads a server or raises an exception.""" client = create_nova_client(context) try: @@ -108,6 +110,8 @@ class InstanceStatus(object): BLOCKED = "BLOCKED" BUILD = "BUILD" FAILED = "FAILED" + REBOOT = "REBOOT" + RESIZE = "RESIZE" SHUTDOWN = "SHUTDOWN" ERROR = "ERROR" @@ -120,10 +124,22 @@ SERVER_INVALID_ACTION_STATUSES = ["BUILD", "REBOOT", "REBUILD"] VALID_ACTION_STATUSES = ["ACTIVE"] -class Instance(object): +def ExecuteInstanceMethod(context, id, method_name, *args, **kwargs): + """Loads an instance and executes a method.""" + arg_str = utils.create_method_args_string(*args, **kwargs) + LOG.debug("Loading instance %s to make the following call: %s(%s)." + % (id, method_name, arg_str)) + instance = Instance.load(context, id) + func = getattr(instance, method_name) + func(*args, **kwargs) - _data_fields = ['name', 'status', 'id', 'created', 'updated', - 'flavor', 'links', 'addresses', 'volume'] + +class Instance(object): + """Represents an instance. + + The life span of this object should be limited. Do not store them or + pass them between threads. + """ def __init__(self, context, db_info, server, service_status, volumes): self.context = context @@ -132,6 +148,16 @@ class Instance(object): self.service_status = service_status self.volumes = volumes + def call_async(self, method, *args, **kwargs): + """Calls a method on this instance in the background and returns. + + This will be a call to some module similar to the guest API, but for + now we just call the real method in eventlet. + + """ + eventlet.spawn(ExecuteInstanceMethod, self.context, self.db_info.id, + method.__name__, *args, **kwargs) + @staticmethod def load(context, id): if context is None: @@ -142,8 +168,8 @@ class Instance(object): db_info = DBInstance.find_by(id=id) except rd_exceptions.NotFound: raise rd_exceptions.NotFound(uuid=id) - server, volumes = load_server(context, db_info.id, - db_info.compute_instance_id) + server, volumes = load_server_with_volumes(context, db_info.id, + db_info.compute_instance_id) task_status = db_info.task_status service_status = InstanceServiceStatus.find_by(instance_id=id) LOG.info("service status=%s" % service_status) @@ -182,6 +208,9 @@ class Instance(object): volume_size, display_name="mysql-%s" % db_info.id, display_description=volume_desc) + # Record the volume ID in case something goes wrong. + db_info.volume_id = volume_ref.id + db_info.save() #TODO(cp16net) this is bad to wait here for the volume create # before returning but this was a quick way to get it working # for now we need this to go into the task manager @@ -193,8 +222,7 @@ class Instance(object): v_ref = volume_client.volumes.get(volume_ref.id) if v_ref.status in ['error']: - raise rd_exceptions.ReddwarfError( - _("Could not create volume")) + raise rd_exceptions.VolumeCreationFailure() LOG.debug(_("Created volume %s") % v_ref) # The mapping is in the format: # :[]:[]:[] @@ -206,10 +234,10 @@ class Instance(object): # guest. Also in cases for ovz where this is mounted on # the host, that's not going to work for us. block_device = {'vdb': mapping} - volume = [{'id': v_ref.id, + volumes = [{'id': v_ref.id, 'size': v_ref.size}] LOG.debug("block_device = %s" % block_device) - LOG.debug("volume = %s" % volume) + LOG.debug("volume = %s" % volumes) device_path = CONFIG.get('device_path') mount_point = CONFIG.get('mount_point') @@ -220,12 +248,16 @@ class Instance(object): block_device = None device_path = None mount_point = None - volume = None + volumes = None #end volume_support + #block_device = "" + #device_path = /dev/vdb + #mount_point = /var/lib/mysql volume_info = {'block_device': block_device, 'device_path': device_path, - 'mount_point': mount_point} - return volume, volume_info + 'mount_point': mount_point, + 'volumes': volumes} + return volume_info @classmethod def create(cls, context, name, flavor_ref, image_id, @@ -233,15 +265,25 @@ class Instance(object): db_info = DBInstance.create(name=name, task_status=InstanceTasks.NONE) LOG.debug(_("Created new Reddwarf instance %s...") % db_info.id) - volume, volume_info = cls._create_volume(context, - db_info, - volume_size) + + if volume_size: + volume_info = cls._create_volume(context, db_info, volume_size) + block_device_mapping = volume_info['block_device'] + device_path=volume_info['device_path'] + mount_point=volume_info['mount_point'] + volumes = volume_info['volumes'] + else: + block_device_mapping = None + device_path=None + mount_point=None + volumes = [] + client = create_nova_client(context) files = {"/etc/guest_info": "guest_id=%s\nservice_type=%s\n" % (db_info.id, service_type)} server = client.servers.create(name, image_id, flavor_ref, files=files, - block_device_mapping=volume_info['block_device']) + block_device_mapping=block_device_mapping) LOG.debug(_("Created new compute instance %s.") % server.id) db_info.compute_instance_id = server.id @@ -255,9 +297,9 @@ class Instance(object): # populate the databases model_schemas = populate_databases(databases) guest.prepare(512, model_schemas, users=[], - device_path=volume_info['device_path'], - mount_point=volume_info['mount_point']) - return Instance(context, db_info, server, service_status, volume) + device_path=device_path, + mount_point=mount_point) + return Instance(context, db_info, server, service_status, volumes) def get_guest(self): return create_guest_client(self.context, self.db_info.id) @@ -285,13 +327,15 @@ class Instance(object): # timeouts determine if the task_status should be integrated here # or removed entirely. if InstanceTasks.REBOOTING == self.db_info.task_status: - return "REBOOT" + return InstanceStatus.REBOOT + if InstanceTasks.RESIZING == self.db_info.task_status: + return InstanceStatus.RESIZE # If the server is in any of these states they take precedence. if self.server.status in ["BUILD", "ERROR", "REBOOT", "RESIZE"]: return self.server.status # The service is only paused during a reboot. if ServiceStatuses.PAUSED == self.service_status.status: - return "REBOOT" + return InstanceStatus.REBOOT # If the service status is NEW, then we are building. if ServiceStatuses.NEW == self.service_status.status: return InstanceStatus.BUILD @@ -347,12 +391,77 @@ class Instance(object): LOG.debug(_(msg) % self.status) raise rd_exceptions.UnprocessableEntity(_(msg) % self.status) + def _refresh_compute_server_info(self): + """Refreshes the compute server field.""" + server, volumes = load_server_with_volumes(self.context, + self.db_info.id, self.db_info.compute_instance_id) + self.server = server + self.volumes = volumes + return server + def resize_flavor(self, new_flavor_id): - LOG.info("Resizing flavor of instance %s..." % self.id) - # TODO(tim.simpson): Validate the new flavor ID can be found or - # raise FlavorNotFound exception. - # TODO(tim.simpson): Actually perform flavor resize. - raise RuntimeError("Not implemented (yet).") + self.validate_can_perform_resize() + LOG.debug("resizing instance %s flavor to %s" + % (self.id, new_flavor_id)) + # Validate that the flavor can be found and that it isn't the same size + # as the current one. + client = create_nova_client(self.context) + try: + new_flavor = client.flavors.get(new_flavor_id) + except nova_exceptions.NotFound: + raise rd_exceptions.FlavorNotFound(uuid=new_flavor_id) + old_flavor = client.flavors.get(self.server.flavor['id']) + new_flavor_size = new_flavor.ram + old_flavor_size = old_flavor.ram + if new_flavor_size == old_flavor_size: + raise rd_exceptions.CannotResizeToSameSize() + + # Set the task to RESIZING and begin the async call before returning. + self.db_info.task_status = InstanceTasks.RESIZING + self.db_info.save() + LOG.debug("Instance %s set to RESIZING." % self.id) + self.call_async(self.resize_flavor_async, new_flavor_id, + old_flavor_size, new_flavor_size) + + def resize_flavor_async(self, new_flavor_id, old_memory_size, + updated_memory_size): + try: + LOG.debug("Instance %s calling stop_mysql..." % self.id) + guest = create_guest_client(self.context, self.db_info.id) + guest.stop_mysql() + try: + LOG.debug("Instance %s calling Compute resize..." % self.id) + self.server.resize(new_flavor_id) + #TODO(tim.simpson): Figure out some way to message the + # following exceptions: + # nova_exceptions.NotFound (for the flavor) + # nova_exceptions.OverLimit + + # Wait for the flavor to change. + #TODO(tim.simpson): Bring back our good friend poll_until. + while(self.server.status == "RESIZE" or + str(self.flavor['id']) != str(new_flavor_id)): + time.sleep(1) + info = self._refresh_compute_server_info() + # Confirm the resize with Nova. + LOG.debug("Instance %s calling Compute confirm resize..." + % self.id) + self.server.confirm_resize() + except Exception as ex: + updated_memory_size = old_memory_size + LOG.error("Error during resize compute! Aborting action.") + LOG.error(ex) + raise + finally: + # Tell the guest to restart MySQL with the new RAM size. + # This is in the finally because we have to call this, or + # else MySQL could stay turned off on an otherwise usable + # instance. + LOG.debug("Instance %s starting mysql..." % self.id) + guest.start_mysql_with_conf_changes(updated_memory_size) + finally: + self.db_info.task_status = InstanceTasks.NONE + self.db_info.save() def resize_volume(self, new_size): LOG.info("Resizing volume of instance %s..." % self.id) @@ -539,7 +648,8 @@ class DBInstance(DatabaseModelBase): #TODO(tim.simpson): Add start time. _data_fields = ['name', 'created', 'compute_instance_id', - 'task_id', 'task_description', 'task_start_time'] + 'task_id', 'task_description', 'task_start_time', + 'volume_id'] def __init__(self, task_status=None, **kwargs): kwargs["task_id"] = task_status.code diff --git a/reddwarf/instance/service.py b/reddwarf/instance/service.py index 176f39a329..80c639cd0b 100644 --- a/reddwarf/instance/service.py +++ b/reddwarf/instance/service.py @@ -46,6 +46,7 @@ class BaseController(wsgi.Controller): webob.exc.HTTPBadRequest: [ models.InvalidModelError, exception.BadRequest, + exception.CannotResizeToSameSize, ], webob.exc.HTTPNotFound: [ exception.NotFound, @@ -53,6 +54,9 @@ class BaseController(wsgi.Controller): ], webob.exc.HTTPConflict: [ ], + webob.exc.HTTPRequestEntityTooLarge: [ + exception.OverLimit, + ] } def __init__(self): @@ -149,7 +153,7 @@ class InstanceController(BaseController): raise rd_exceptions.BadRequest("Invalid resize argument %s" % key) if selected_option: - return selected_option(self, instance, args) + return selected_option(instance, args) else: raise rd_exceptions.BadRequest(_("Missing resize arguments.")) @@ -258,7 +262,10 @@ class InstanceController(BaseController): databases = body['instance'].get('databases') if databases is None: databases = [] - volume_size = body['instance']['volume']['size'] + if body['instance'].get('volume', None) is not None: + volume_size = body['instance']['volume']['size'] + else: + volume_size = None instance = models.Instance.create(context, name, flavor_ref, image_id, databases, service_type, volume_size) @@ -280,8 +287,8 @@ class InstanceController(BaseController): volume_size = float(size) except (ValueError, TypeError) as err: LOG.error(err) - msg = ("Required element/key - instance volume" - "'size' was not specified as a number") + msg = ("Required element/key - instance volume 'size' was not " + "specified as a number (value was %s)." % size) raise rd_exceptions.ReddwarfError(msg) if int(volume_size) != volume_size or int(volume_size) < 1: msg = ("Volume 'size' needs to be a positive " @@ -304,12 +311,13 @@ class InstanceController(BaseController): body['instance'] body['instance']['flavorRef'] # TODO(cp16net) add in volume to the mix - volume_size = body['instance']['volume']['size'] + if 'volume' in body['instance'] and \ + body['instance']['volume'] is not None: + volume_size = body['instance']['volume']['size'] except KeyError as e: LOG.error(_("Create Instance Required field(s) - %s") % e) raise rd_exceptions.ReddwarfError("Required element/key - %s " "was not specified" % e) - InstanceController._validate_volume_size(volume_size) @staticmethod def _validate_resize_instance(body): diff --git a/reddwarf/instance/tasks.py b/reddwarf/instance/tasks.py index 9f8c60c227..dafb2cbdbb 100644 --- a/reddwarf/instance/tasks.py +++ b/reddwarf/instance/tasks.py @@ -58,6 +58,7 @@ class InstanceTasks(object): NONE = InstanceTask(0x01, 'NONE') DELETING = InstanceTask(0x02, 'DELETING') REBOOTING = InstanceTask(0x03, 'REBOOTING') + RESIZING = InstanceTask(0x04, 'RESIZING') # Dissuade further additions at run-time. diff --git a/reddwarf/tests/fakes/guestagent.py b/reddwarf/tests/fakes/guestagent.py index 6b3f21d667..3a47d88ea0 100644 --- a/reddwarf/tests/fakes/guestagent.py +++ b/reddwarf/tests/fakes/guestagent.py @@ -89,23 +89,16 @@ class FakeGuest(object): def start_mysql_with_conf_changes(self, updated_memory_size): from reddwarf.instance.models import InstanceServiceStatus from reddwarf.instance.models import ServiceStatuses - - def update_db(): - status = InstanceServiceStatus.find_by(instance_id=self.id) - status.status = ServiceStatuses.RUNNING - status.save() - EventSimulator.add_event(0.5, update_db) + status = InstanceServiceStatus.find_by(instance_id=self.id) + status.status = ServiceStatuses.RUNNING + status.save() def stop_mysql(self): from reddwarf.instance.models import InstanceServiceStatus from reddwarf.instance.models import ServiceStatuses - - def update_db(): - status = InstanceServiceStatus.find_by(instance_id=self.id) - status.status = ServiceStatuses.SHUTDOWN - status.save() - EventSimulator.add_event(0.5, update_db) - + status = InstanceServiceStatus.find_by(instance_id=self.id) + status.status = ServiceStatuses.SHUTDOWN + status.save() def get_or_create(id): if id not in DB: diff --git a/reddwarf/tests/fakes/nova.py b/reddwarf/tests/fakes/nova.py index 170e307ea8..a28b3660fe 100644 --- a/reddwarf/tests/fakes/nova.py +++ b/reddwarf/tests/fakes/nova.py @@ -59,12 +59,14 @@ class FakeFlavors(object): self._add(3, 10, "m1.medium", 4096) self._add(4, 10, "m1.large", 8192) self._add(5, 10, "m1.xlarge", 16384) + self._add(6, 0, "tinier", 506) def _add(self, *args, **kwargs): new_flavor = FakeFlavor(*args, **kwargs) self.db[new_flavor.id] = new_flavor def get(self, id): + id = int(id) if id not in self.db: raise nova_exceptions.NotFound(404, "Flavor id not found %s" % id) return self.db[id] @@ -85,7 +87,7 @@ class FakeFlavors(object): class FakeServer(object): def __init__(self, parent, owner, id, name, image_id, flavor_ref, - block_device_mapping): + block_device_mapping, volumes): self.owner = owner # This is a context. self.id = id self.parent = parent @@ -94,13 +96,18 @@ class FakeServer(object): self.flavor_ref = flavor_ref self.events = EventSimulator() self.schedule_status("BUILD", 0.0) - LOG.debug("block_device_mapping = %s" % block_device_mapping) - self.block_device_mapping = block_device_mapping + self.volumes = volumes @property def addresses(self): return {"private": [{"addr":"123.123.123.123"}]} + def confirm_resize(self): + if self.status != "VERIFY_RESIZE": + raise RuntimeError("Not in resize confirm mode.") + self._current_status = "ACTIVE" + + def delete(self): self.schedule_status = [] self._current_status = "SHUTDOWN" @@ -117,6 +124,16 @@ class FakeServer(object): "rel": link_type } for link_type in ['self', 'bookmark']] + def resize(self, new_flavor_id): + self._current_status = "RESIZE" + def set_to_confirm_mode(): + self._current_status = "VERIFY_RESIZE" + def set_flavor(): + flavor = self.parent.flavors.get(new_flavor_id) + self.flavor_ref = flavor.links[0]['href'] + self.events.add_event(1, set_to_confirm_mode) + self.events.add_event(1, set_flavor) + def schedule_status(self, new_status, time_from_now): """Makes a new status take effect at the given time.""" def set_status(): @@ -159,13 +176,32 @@ class FakeServers(object): def create(self, name, image_id, flavor_ref, files, block_device_mapping): id = "FAKE_%d" % self.next_id self.next_id += 1 + volumes = self._get_volumes_from_bdm(block_device_mapping) server = FakeServer(self, self.context, id, name, image_id, flavor_ref, - block_device_mapping) + block_device_mapping, volumes) self.db[id] = server server.schedule_status("ACTIVE", 1) LOG.info("FAKE_SERVERS_DB : %s" % str(FAKE_SERVERS_DB)) return server + def _get_volumes_from_bdm(self, block_device_mapping): + volumes = [] + if block_device_mapping is not None: + # block_device_mapping is a dictionary, where the key is the + # device name on the compute instance and the mapping info is a + # set of fields in a string, seperated by colons. + # For each device, find the volume, and record the mapping info + # to another fake object and attach it to the volume + # so that the fake API can later retrieve this. + for device in block_device_mapping: + mapping = block_device_mapping[device] + (id, _type, size, delete_on_terminate) = mapping.split(":") + volume = self.volumes.get(id) + volume.mapping = FakeBlockDeviceMappingInfo(id, device, + _type, size, delete_on_terminate) + volumes.append(volume) + return volumes + def get(self, id): if id not in self.db: LOG.error("Couldn't find server id %s, collection=%s" % (id, @@ -177,6 +213,11 @@ class FakeServers(object): else: raise nova_exceptions.NotFound(404, "Bad permissions") + def get_server_volumes(self, server_id): + return [volume.mapping + for volume in self.get(server_id).volumes + if volume.mapping is not None] + def list(self): return [v for (k, v) in self.db.items() if self.can_see(v.id)] @@ -206,20 +247,8 @@ class FakeServerVolumes(object): return [ServerVolumes(server.block_device_mapping)] -FLAVORS = FakeFlavors() -class FakeClient(object): - - def __init__(self, context): - self.context = context - self.flavors = FLAVORS - self.servers = FakeServers(context, self.flavors) - self.volumes = FakeServerVolumes(context) - - -def fake_create_nova_client(context): - return FakeClient(context) class FakeVolume(object): @@ -251,6 +280,16 @@ class FakeVolume(object): return self._current_status +class FakeBlockDeviceMappingInfo(object): + + def __init__(self, id, device, _type, size, delete_on_terminate): + self.volumeId = id + self.device = device + self.type = _type + self.size = size + self.delete_on_terminate = delete_on_terminate + + FAKE_VOLUMES_DB = {} @@ -290,12 +329,40 @@ class FakeVolumes(object): return volume -class FakeVolumeClient(object): +FLAVORS = FakeFlavors() + +class FakeClient(object): def __init__(self, context): self.context = context + self.flavors = FLAVORS + self.servers = FakeServers(context, self.flavors) self.volumes = FakeVolumes(context) + self.servers.volumes = self.volumes + + def get_server_volumes(self, server_id): + return self.servers.get_server_volumes(server_id) + + +CLIENT_DATA = {} + + +def get_client_data(context): + if context not in CLIENT_DATA: + nova_client = FakeClient(context) + volume_client = FakeClient(context) + nova_client.volumes = volume_client + volume_client.servers = nova_client + CLIENT_DATA[context] = { + 'nova': nova_client, + 'volume': volume_client + } + return CLIENT_DATA[context] + + +def fake_create_nova_client(context): + return get_client_data(context)['nova'] def fake_create_nova_volume_client(context): - return FakeVolumeClient(context) + return get_client_data(context)['volume']