From 311022f35e8a42b2e249606ce828e303d4a417eb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 29 Mar 2012 11:48:34 -0700 Subject: [PATCH 1/8] Adding checks for interfaces before we install, instead of during --- devstack/components/nova.py | 77 ++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/devstack/components/nova.py b/devstack/components/nova.py index 3d8e5923..7a48a323 100644 --- a/devstack/components/nova.py +++ b/devstack/components/nova.py @@ -256,11 +256,13 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent): self.bin_dir = sh.joinpths(self.app_dir, BIN_DIR) self.paste_conf_fn = self._get_target_config_name(PASTE_CONF) self.volumes_enabled = False - if NVOL in self.desired_subsystems: - self.volumes_enabled = True - self.xvnc_enabled = False - if NXVNC in self.desired_subsystems: - self.xvnc_enabled = True + self.volume_configurator = None + self.volumes_enabled = NVOL in self.desired_subsystems + self.xvnc_enabled = NXVNC in self.desired_subsystems + self.volume_maker = None + if self.volumes_enabled: + self.volume_maker = NovaVolumeConfigurator(self) + self.conf_maker = NovaConfConfigurator(self) def _get_symlinks(self): links = comp.PythonInstallComponent._get_symlinks(self) @@ -268,6 +270,12 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent): links[source_fn] = sh.joinpths(self._get_link_dir(), API_CONF) return links + def verify(self): + comp.PythonInstallComponent.verify(self) + self.conf_maker.verify() + if self.volume_maker: + self.volume_maker.verify() + def warm_configs(self): warm_pws = list(WARMUP_PWS) driver_canon = canon_virt_driver(self.cfg.get('nova', 'virt_driver')) @@ -300,9 +308,8 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent): self._setup_cleaner() self._setup_network_initer() # Check if we need to do the vol subsystem - if self.volumes_enabled: - vol_maker = NovaVolumeConfigurator(self) - vol_maker.setup_volumes() + if self.volume_maker: + self.volume_maker.setup_volumes() def _setup_cleaner(self): LOG.info("Configuring cleaner template %r", CLEANER_DATA_CONF) @@ -319,12 +326,9 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent): db.create_db(self.cfg, self.pw_gen, self.distro, DB_NAME) def _generate_nova_conf(self): - LOG.info("Generating dynamic content for nova in file %r" % (API_CONF)) - conf_gen = NovaConfConfigurator(self) - nova_conf_contents = conf_gen.configure() conf_fn = self._get_target_config_name(API_CONF) - LOG.info("Writing nova configuration to %r" % (conf_fn)) - LOG.debug(nova_conf_contents) + LOG.info("Generating dynamic content for nova: %r" % (conf_fn)) + nova_conf_contents = self.conf_maker.configure() self.tracewriter.dirs_made(*sh.mkdirslist(sh.dirname(conf_fn))) self.tracewriter.cfg_file_written(sh.write_file(conf_fn, nova_conf_contents)) @@ -434,6 +438,9 @@ class NovaVolumeConfigurator(object): def setup_volumes(self): self._setup_vol_groups() + def verify(self): + pass + def _setup_vol_groups(self): LOG.info("Attempting to setup volume groups for nova volume management.") mp = dict() @@ -520,6 +527,29 @@ class NovaConfConfigurator(object): def _getstr(self, name, default=''): return self.cfg.getdefaulted('nova', name, default) + def verify(self): + # Do a little check to make sure actually have that interface/s + public_interface = self._getstr('public_interface') + vlan_interface = self._getstr('vlan_interface', public_interface) + if not utils.is_interface(public_interface): + msg = "Public interface %r is not a known interface" % (public_interface) + raise exceptions.ConfigException(msg) + if not utils.is_interface(vlan_interface): + msg = "VLAN interface %r is not a known interface" % (vlan_interface) + raise exceptions.ConfigException(msg) + # Driver specific interface checks + drive_canon = canon_virt_driver(self._getstr('virt_driver')) + if drive_canon == 'xenserver': + xs_flat_ifc = self._getstr('xs_flat_interface', XS_DEF_INTERFACE) + if xs_flat_ifc and not utils.is_interface(xs_flat_ifc): + msg = "Xenserver flat interface %s is not a known interface" % (xs_flat_ifc) + raise exceptions.ConfigException(msg) + elif drive_canon == 'libvirt': + flat_interface = self._getstr('flat_interface') + if flat_interface and not utils.is_interface(flat_interface: + msg = "Libvirt flat interface %s is not a known interface" % (flat_interface) + raise exceptions.ConfigException(msg) + def configure(self): # Everything built goes in here nova_conf = NovaConf() @@ -737,16 +767,6 @@ class NovaConfConfigurator(object): # of public_interface. We'll grab the value and keep it handy. public_interface = self._getstr('public_interface') vlan_interface = self._getstr('vlan_interface', public_interface) - - # Do a little check to make sure actually have that interface/s - if not utils.is_interface(public_interface): - msg = "Public interface %r is not a known interface" % (public_interface) - raise exceptions.ConfigException(msg) - - if not utils.is_interface(vlan_interface): - msg = "VLAN interface %r is not a known interface" % (vlan_interface) - raise exceptions.ConfigException(msg) - nova_conf.add('public_interface', public_interface) nova_conf.add('vlan_interface', vlan_interface) @@ -796,21 +816,16 @@ class NovaConfConfigurator(object): nova_conf.add('xenapi_connection_username', self._getstr('xa_connection_username', XA_DEF_USER)) nova_conf.add('xenapi_connection_password', self.cfg.get("passwords", "xenapi_connection")) nova_conf.add('noflat_injected', True) - xs_flat_ifc = self._getstr('xs_flat_interface', XS_DEF_INTERFACE) - if not utils.is_interface(xs_flat_ifc): - msg = "Xenserver flat interface %s is not a known interface" % (xs_flat_ifc) - raise exceptions.ConfigException(msg) - nova_conf.add('flat_interface', xs_flat_ifc) nova_conf.add('firewall_driver', FIRE_MANAGER_TEMPLATE % (self._getstr('xs_firewall_driver', DEF_FIREWALL_DRIVER))) nova_conf.add('flat_network_bridge', self._getstr('xs_flat_network_bridge', XS_DEF_BRIDGE)) + xs_flat_ifc = self._getstr('xs_flat_interface', XS_DEF_INTERFACE) + if xs_flat_ifc: + nova_conf.add('flat_interface', xs_flat_ifc) elif drive_canon == 'libvirt': nova_conf.add('firewall_driver', FIRE_MANAGER_TEMPLATE % (self._getstr('libvirt_firewall_driver', DEF_FIREWALL_DRIVER))) nova_conf.add('flat_network_bridge', self._getstr('flat_network_bridge', DEF_FLAT_VIRT_BRIDGE)) flat_interface = self._getstr('flat_interface') if flat_interface: - if not utils.is_interface(flat_interface): - msg = "Libvirt flat interface %s is not a known interface" % (flat_interface) - raise exceptions.ConfigException(msg) nova_conf.add('flat_interface', flat_interface) From 28b1068d7d221ed78a5fa7d40bb7234459f0c963 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 29 Mar 2012 11:49:32 -0700 Subject: [PATCH 2/8] Fixed syntax, oops --- devstack/components/nova.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devstack/components/nova.py b/devstack/components/nova.py index 7a48a323..1e103f27 100644 --- a/devstack/components/nova.py +++ b/devstack/components/nova.py @@ -546,7 +546,7 @@ class NovaConfConfigurator(object): raise exceptions.ConfigException(msg) elif drive_canon == 'libvirt': flat_interface = self._getstr('flat_interface') - if flat_interface and not utils.is_interface(flat_interface: + if flat_interface and not utils.is_interface(flat_interface): msg = "Libvirt flat interface %s is not a known interface" % (flat_interface) raise exceptions.ConfigException(msg) From 79104bc3a8bbeb2db71da7a4b91676ce42a5ec69 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 29 Mar 2012 12:31:39 -0700 Subject: [PATCH 3/8] Updating so we don't try to update git repos if they exist, leaving them alone Updating so that we don't uninstall python packages if keep old --- conf/distros/fedora-16.yaml | 1 - conf/distros/rhel-6.yaml | 1 - conf/distros/ubuntu-oneiric.yaml | 1 - devstack/component.py | 21 +++++++++++++-------- devstack/components/keystone.py | 2 +- devstack/downloader.py | 11 +++-------- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/conf/distros/fedora-16.yaml b/conf/distros/fedora-16.yaml index 8f79a026..5025e91d 100644 --- a/conf/distros/fedora-16.yaml +++ b/conf/distros/fedora-16.yaml @@ -13,7 +13,6 @@ commands: git: checkout: git checkout clone: git clone - pull: git pull libvirt: restart: service libvirtd restart status: service libvirtd status diff --git a/conf/distros/rhel-6.yaml b/conf/distros/rhel-6.yaml index ab3b9cd3..9c6390ee 100644 --- a/conf/distros/rhel-6.yaml +++ b/conf/distros/rhel-6.yaml @@ -14,7 +14,6 @@ commands: git: checkout: git checkout clone: git clone - pull: git pull libvirt: restart: service libvirtd restart status: service libvirtd status diff --git a/conf/distros/ubuntu-oneiric.yaml b/conf/distros/ubuntu-oneiric.yaml index 61716006..ee2b9f85 100644 --- a/conf/distros/ubuntu-oneiric.yaml +++ b/conf/distros/ubuntu-oneiric.yaml @@ -14,7 +14,6 @@ commands: git: checkout: git checkout clone: git clone - pull: git pull iscsi: restart: service tgt restart start: service tgt start diff --git a/devstack/component.py b/devstack/component.py index 53224658..a33d9c49 100644 --- a/devstack/component.py +++ b/devstack/component.py @@ -469,14 +469,16 @@ class PkgUninstallComponent(ComponentBase, PackageBasedComponentMixin): dirs_made = [sh.abspth(d) for d in dirs_made] if self.keep_old: download_places = [path_location[0] for path_location in self.tracereader.download_locations()] - utils.log_iterable(download_places, logger=LOG, - header="Keeping %s download directories" % (len(download_places))) - for download_place in download_places: - dirs_made = sh.remove_parents(download_place, dirs_made) - utils.log_iterable(dirs_made, logger=LOG, - header="Removing %s created directories" % (len(dirs_made))) - for dir_name in dirs_made: - sh.deldir(dir_name, run_as_root=True) + if download_places: + utils.log_iterable(download_places, logger=LOG, + header="Keeping %s download directories (and there children directories)" % (len(download_places))) + for download_place in download_places: + dirs_made = sh.remove_parents(download_place, dirs_made) + if dirs_made: + utils.log_iterable(dirs_made, logger=LOG, + header="Removing %s created directories" % (len(dirs_made))) + for dir_name in dirs_made: + sh.deldir(dir_name, run_as_root=True) class PythonUninstallComponent(PkgUninstallComponent): @@ -489,6 +491,9 @@ class PythonUninstallComponent(PkgUninstallComponent): PkgUninstallComponent.uninstall(self) def _uninstall_pips(self): + if self.keep_old: + LOG.info('Keep-old flag set, not removing any python packages.') + return pips = self.tracereader.pips_installed() if pips: pip_names = set([p['name'] for p in pips]) diff --git a/devstack/components/keystone.py b/devstack/components/keystone.py index 7313457b..02fbcd2f 100644 --- a/devstack/components/keystone.py +++ b/devstack/components/keystone.py @@ -126,7 +126,7 @@ class KeystoneInstaller(comp.PythonInstallComponent): db.create_db(self.cfg, self.pw_gen, self.distro, DB_NAME) def _setup_initer(self): - LOG.info("Configuring keystone initializer template %s.", MANAGE_DATA_CONF) + LOG.info("Configuring keystone initializer template %r", MANAGE_DATA_CONF) (_, contents) = utils.load_template(self.component_name, MANAGE_DATA_CONF) mp = self._get_param_map(MANAGE_DATA_CONF) contents = utils.param_replace(contents, mp, True) diff --git a/devstack/downloader.py b/devstack/downloader.py index 6b0a7337..cbd869c5 100644 --- a/devstack/downloader.py +++ b/devstack/downloader.py @@ -48,20 +48,15 @@ class GitDownloader(Downloader): def download(self): dirsmade = list() if sh.isdir(self.store_where): - LOG.info("Updating using git: located at %r" % (self.store_where)) - cmd = list(self.distro.get_command('git', 'checkout')) - cmd += [GIT_MASTER_BRANCH] - sh.execute(*cmd, cwd=self.store_where) - cmd = self.distro.get_command('git', 'pull') - sh.execute(*cmd, cwd=self.store_where) + LOG.info("Existing directory located at %r, leaving it alone." % (self.store_where)) else: - LOG.info("Downloading using git: %r to %r" % (self.uri, self.store_where)) + LOG.info("Downloading %r to %r" % (self.uri, self.store_where)) dirsmade.extend(sh.mkdirslist(self.store_where)) cmd = list(self.distro.get_command('git', 'clone')) cmd += [self.uri, self.store_where] sh.execute(*cmd) if self.branch and self.branch != GIT_MASTER_BRANCH: - LOG.info("Adjusting branch using git: %r" % (self.branch)) + LOG.info("Adjusting branch to %r" % (self.branch)) cmd = list(self.distro.get_command('git', 'checkout')) cmd += [self.branch] sh.execute(*cmd, cwd=self.store_where) From 63e0384de5eefb006940dc6ac4859f8197dd5388 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 29 Mar 2012 13:23:58 -0700 Subject: [PATCH 4/8] More string fixes --- devstack/env.py | 9 +++++---- devstack/env_rc.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/devstack/env.py b/devstack/env.py index 29d7eed1..7c5c2c2d 100644 --- a/devstack/env.py +++ b/devstack/env.py @@ -30,18 +30,19 @@ def set(key, value): # See: from http://docs.python.org/library/os.html # Calling putenv() directly does not change os.environ, so it's better to modify os.environ. if key is not None: - LOG.audit("Setting environment key [%s] to value [%s]" % (key, value)) + LOG.audit("Setting environment key %r to value %r" % (str(key), str(value))) os.environ[str(key)] = str(value) def get_key(key, default_value=None): if not key: return default_value - LOG.debug("Looking up environment variable [%s]" % (key)) + key = str(key) + LOG.debug("Looking up environment variable %r" % (key)) value = get().get(key) if value is None: - LOG.debug("Could not find anything in environment variable [%s]" % (key)) + LOG.debug("Could not find anything in environment variable %r" % (key)) value = default_value else: - LOG.audit("Found [%s] in environment variable [%s]" % (value, key)) + LOG.audit("Found %r in environment variable %r" % (value, key)) return value diff --git a/devstack/env_rc.py b/devstack/env_rc.py index e9c0755d..97a90ea9 100644 --- a/devstack/env_rc.py +++ b/devstack/env_rc.py @@ -272,12 +272,12 @@ class RcReader(object): def extract(self, fn): extracted_vars = dict() contents = '' - LOG.audit("Loading rc file [%s]" % (fn)) + LOG.audit("Loading rc file %r" % (fn)) try: with open(fn, 'r') as fh: contents = fh.read() except IOError as e: - LOG.warn("Failed extracting rc file [%s] due to [%s]" % (fn, e)) + LOG.warn("Failed extracting rc file %r due to %s" % (fn, e)) return extracted_vars for line in contents.splitlines(): if self._is_comment(line): From 992a5c78af24fc3f93cb56cc807e3489080a1e4b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 29 Mar 2012 13:32:29 -0700 Subject: [PATCH 5/8] Setup mixin for novnc component --- devstack/components/nova.py | 4 ++-- devstack/components/novnc.py | 32 +++++++++++++++++--------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/devstack/components/nova.py b/devstack/components/nova.py index 1e103f27..63f512cf 100644 --- a/devstack/components/nova.py +++ b/devstack/components/nova.py @@ -547,8 +547,8 @@ class NovaConfConfigurator(object): elif drive_canon == 'libvirt': flat_interface = self._getstr('flat_interface') if flat_interface and not utils.is_interface(flat_interface): - msg = "Libvirt flat interface %s is not a known interface" % (flat_interface) - raise exceptions.ConfigException(msg) + msg = "Libvirt flat interface %s is not a known interface" % (flat_interface) + raise exceptions.ConfigException(msg) def configure(self): # Everything built goes in here diff --git a/devstack/components/novnc.py b/devstack/components/novnc.py index 977d77be..c10b3af6 100644 --- a/devstack/components/novnc.py +++ b/devstack/components/novnc.py @@ -33,17 +33,9 @@ APP_OPTIONS = { } -class NoVNCUninstaller(comp.PythonUninstallComponent): - def __init__(self, *args, **kargs): - comp.PythonUninstallComponent.__init__(self, *args, **kargs) - - -class NoVNCInstaller(comp.PythonInstallComponent): - def __init__(self, *args, **kargs): - comp.PythonInstallComponent.__init__(self, *args, **kargs) - - def _get_python_directories(self): - return dict() +class NoVNCMixin(object): + def known_options(self): + return set(['nova']) def _get_download_locations(self): places = list() @@ -54,7 +46,20 @@ class NoVNCInstaller(comp.PythonInstallComponent): return places -class NoVNCRuntime(comp.ProgramRuntime): +class NoVNCUninstaller(NoVNCMixin, comp.PythonUninstallComponent): + def __init__(self, *args, **kargs): + comp.PythonUninstallComponent.__init__(self, *args, **kargs) + + +class NoVNCInstaller(NoVNCMixin, comp.PythonInstallComponent): + def __init__(self, *args, **kargs): + comp.PythonInstallComponent.__init__(self, *args, **kargs) + + def _get_python_directories(self): + return dict() + + +class NoVNCRuntime(NoVNCMixin, comp.ProgramRuntime): def __init__(self, *args, **kargs): comp.ProgramRuntime.__init__(self, *args, **kargs) @@ -67,9 +72,6 @@ class NoVNCRuntime(comp.ProgramRuntime): }) return apps - def known_options(self): - return set(['nova']) - def _get_param_map(self, app_name): root_params = comp.ProgramRuntime._get_param_map(self, app_name) if app_name == VNC_PROXY_APP and 'nova' in self.options: From 1405685d2e203c149792dfd00b799c15a1c98190 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 29 Mar 2012 13:41:18 -0700 Subject: [PATCH 6/8] Use ~/openstack as root directory if not specified and not found in the env --- stack | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stack b/stack index ef1accb9..c3a6e445 100755 --- a/stack +++ b/stack @@ -118,8 +118,7 @@ def run(args): loaded_rcs = True root_dir = env.get_key(env_rc.INSTALL_ROOT) if not root_dir: - print(utils.color_text("No root directory specified!", "red")) - return False + root_dir = utils.joinpths(sh.gethomedir(), 'openstack') root_dir = sh.abspth(root_dir) setup_root(root_dir) From 44f1ed062f06d4e56e9707b46bacfd69b1a910a6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 29 Mar 2012 15:45:06 -0700 Subject: [PATCH 7/8] Simplify image uploading and allow for qcow2 uploading --- devstack/image/uploader.py | 149 +++++++++++++++++++++---------------- stack | 2 +- 2 files changed, 84 insertions(+), 67 deletions(-) diff --git a/devstack/image/uploader.py b/devstack/image/uploader.py index 9abc4c00..e071cd18 100644 --- a/devstack/image/uploader.py +++ b/devstack/image/uploader.py @@ -32,21 +32,17 @@ from devstack.components import keystone LOG = log.getLogger("devstack.image.uploader") # These are used when looking inside archives -KERNEL_FN_MATCH = re.compile(r"(.*)vmlinuz$", re.I) -RAMDISK_FN_MATCH = re.compile(r"(.*)initrd$", re.I) +KERNEL_FN_MATCH = re.compile(r"(.*)-vmlinuz$", re.I) +RAMDISK_FN_MATCH = re.compile(r"(.*)-initrd$", re.I) IMAGE_FN_MATCH = re.compile(r"(.*)img$", re.I) # Glance commands -KERNEL_ADD = ['glance', 'add', '-A', '%TOKEN%', '--silent-upload', - 'name="%IMAGE_NAME%-kernel"', 'is_public=true', 'container_format=aki', - 'disk_format=aki'] -INITRD_ADD = ['glance', 'add', '-A', '%TOKEN%', '--silent-upload', - 'name="%IMAGE_NAME%-ramdisk"', 'is_public=true', 'container_format=ari', - 'disk_format=ari'] -IMAGE_ADD = ['glance', 'add', '-A', '%TOKEN%', '--silent-upload', - 'name="%IMAGE_NAME%.img"', - 'is_public=true', 'container_format=ami', 'disk_format=ami', - 'kernel_id=%KERNEL_ID%', 'ramdisk_id=%INITRD_ID%'] +IMAGE_ADD = ['glance', 'add', '-A', '%TOKEN%', + '--silent-upload', + 'name="%NAME%"', + 'is_public=true', + 'container_format=%CONTAINER_FORMAT%', + 'disk_format=%DISK_FORMAT%'] DETAILS_SHOW = ['glance', '-A', '%TOKEN%', 'details'] # Extensions that tarfile knows how to work with @@ -58,6 +54,7 @@ TAR_EXTS = ['.tgz', '.gzip', '.gz', '.bz2', '.tar'] NAME_CLEANUPS = [ '.tar.gz', '.img.gz', + '.qcow2', '.img', ] + TAR_EXTS NAME_CLEANUPS.sort() @@ -96,26 +93,38 @@ class Unpacker(object): LOG.info("Extracting %r to %r", file_location, extract_dir) with contextlib.closing(tarfile.open(file_location, 'r')) as tfh: tfh.extractall(extract_dir) - locations = dict() + info = dict() if kernel_fn: - locations['kernel'] = sh.joinpths(extract_dir, kernel_fn) + info['kernel'] = { + 'FILE_NAME': sh.joinpths(extract_dir, kernel_fn), + 'DISK_FORMAT': 'aki', + 'CONTAINER_FORMAT': 'aki', + } if ramdisk_fn: - locations['ramdisk'] = sh.joinpths(extract_dir, ramdisk_fn) - locations['image'] = sh.joinpths(extract_dir, root_img_fn) - return locations - - def _unpack_image(self, file_name, file_location, tmp_dir): - locations = dict() - locations['image'] = file_location - return locations + info['ramdisk'] = { + 'FILE_NAME': sh.joinpths(extract_dir, ramdisk_fn), + 'DISK_FORMAT': 'ari', + 'CONTAINER_FORMAT': 'ari', + } + info['FILE_NAME'] = sh.joinpths(extract_dir, root_img_fn) + info['DISK_FORMAT'] = 'ami' + info['CONTAINER_FORMAT'] = 'ami' + return info def unpack(self, file_name, file_location, tmp_dir): (_, fn_ext) = os.path.splitext(file_name) fn_ext = fn_ext.lower() if fn_ext in TAR_EXTS: return self._unpack_tar(file_name, file_location, tmp_dir) - elif fn_ext in ['.img']: - return self._unpack_image(file_name, file_location, tmp_dir) + elif fn_ext in ['.img', '.qcow2']: + info = dict() + info['FILE_NAME'] = file_location, + if fn_ext == '.img': + info['DISK_FORMAT'] = 'raw' + else: + info['DISK_FORMAT'] = 'qcow2' + info['CONTAINER_FORMAT'] = 'bare' + return info else: msg = "Currently we do not know how to unpack %r" % (file_name) raise NotImplementedError(msg) @@ -128,16 +137,18 @@ class Image(object): self.token = token self._registry = Registry(token) - def _register(self, image_name, locations): + def _register(self, image_name, location): # Upload the kernel, if we have one - kernel = locations.get('kernel') + kernel = location.pop('kernel', None) kernel_id = '' if kernel: - LOG.info('Adding kernel %r to glance.', kernel) - params = {'TOKEN': self.token, 'IMAGE_NAME': image_name} - cmd = {'cmd': KERNEL_ADD} - with open(kernel, 'r') as fh: + LOG.info('Adding kernel %s to glance.', kernel) + params = dict(kernel) + params['TOKEN'] = self.token + params['NAME'] = "%s-vmlinuz" % (image_name) + cmd = {'cmd': IMAGE_ADD} + with open(params['FILE_NAME'], 'r') as fh: res = utils.execute_template(cmd, params=params, stdin_fh=fh, close_stdin=True) @@ -146,13 +157,15 @@ class Image(object): kernel_id = stdout.split(':')[1].strip() # Upload the ramdisk, if we have one - initrd = locations.get('ramdisk') + initrd = location.pop('ramdisk', None) initrd_id = '' if initrd: - LOG.info('Adding ramdisk %r to glance.', initrd) - params = {'TOKEN': self.token, 'IMAGE_NAME': image_name} - cmd = {'cmd': INITRD_ADD} - with open(initrd, 'r') as fh: + LOG.info('Adding ramdisk %s to glance.', initrd) + params = dict(initrd) + params['TOKEN'] = self.token + params['NAME'] = "%s-initrd" % (image_name) + cmd = {'cmd': IMAGE_ADD} + with open(params['FILE_NAME'], 'r') as fh: res = utils.execute_template(cmd, params=params, stdin_fh=fh, close_stdin=True) @@ -161,16 +174,24 @@ class Image(object): initrd_id = stdout.split(':')[1].strip() # Upload the root, we must have one... - img_id = '' - root_image = locations['image'] - LOG.info('Adding image %r to glance.', root_image) - params = {'TOKEN': self.token, 'IMAGE_NAME': image_name, - 'KERNEL_ID': kernel_id, 'INITRD_ID': initrd_id} - cmd = {'cmd': IMAGE_ADD} - with open(root_image, 'r') as fh: + root_image = dict(location) + LOG.info('Adding image %s to glance.', root_image) + add_cmd = list(IMAGE_ADD) + params = dict(root_image) + params['TOKEN'] = self.token + params['NAME'] = image_name + if kernel_id: + add_cmd += ['kernel_id=%KERNEL_ID%'] + params['KERNEL_ID'] = kernel_id + if initrd_id: + add_cmd += ['ramdisk_id=%INITRD_ID%'] + params['INITRD_ID'] = initrd_id + cmd = {'cmd': add_cmd} + with open(params['FILE_NAME'], 'r') as fh: res = utils.execute_template(cmd, params=params, stdin_fh=fh, close_stdin=True) + img_id = '' if res: (stdout, _) = res[0] img_id = stdout.split(':')[1].strip() @@ -223,9 +244,9 @@ class Image(object): with utils.tempdir() as tdir: fetch_fn = sh.joinpths(tdir, url_fn) down.UrlLibDownloader(self.url, fetch_fn).download() - locations = Unpacker().unpack(url_fn, fetch_fn, tdir) + unpack_info = Unpacker().unpack(url_fn, fetch_fn, tdir) tgt_image_name = self._generate_img_name(url_fn) - self._register(tgt_image_name, locations) + self._register(tgt_image_name, unpack_info) return tgt_image_name else: return None @@ -300,17 +321,16 @@ class Service: }) # Prepare the request - request = urllib2.Request(keystone_token_url) - - # Post body - request.add_data(data) - - # Content type - request.add_header('Content-Type', 'application/json') + headers = { + 'Content-Type': 'application/json' + } + request = urllib2.Request(keystone_token_url, data=data, headers=headers) # Make the request - LOG.info("Getting your token from url [%s], please wait..." % (keystone_token_url)) - LOG.debug("With post json data %s" % (data)) + LOG.info("Getting your token from url %r, please wait..." % (keystone_token_url)) + LOG.debug("With post data %s" % (data)) + LOG.debug("With headers %s" % (headers)) + response = urllib2.urlopen(request) token = json.loads(response.read()) @@ -320,36 +340,33 @@ class Service: not token.get('access') or not type(token.get('access')) is dict or not token.get('access').get('token') or not type(token.get('access').get('token')) is dict or not token.get('access').get('token').get('id')): - msg = "Response from url [%s] did not match expected json format." % (keystone_token_url) + msg = "Response from url %r did not match expected json format." % (keystone_token_url) raise IOError(msg) # Basic checks passed, extract it! tok = token['access']['token']['id'] - LOG.debug("Got token %s" % (tok)) + LOG.debug("Got token %r" % (tok)) return tok def install(self): LOG.info("Setting up any specified images in glance.") # Extract the urls from the config - urls = list() - flat_urls = self.cfg.getdefaulted('img', 'image_urls', []) - expanded_urls = [x.strip() for x in flat_urls.split(',')] - for url in expanded_urls: - if len(url): - urls.append(url) + flat_locations = self.cfg.getdefaulted('img', 'image_urls', '') + locations = [loc.strip() for loc in flat_locations.split(',') if len(loc.strip())] # Install them in glance am_installed = 0 - if urls: - LOG.info("Attempting to download & extract and upload (%s) images." % (", ".join(urls))) + if locations: + utils.log_iterable(locations, logger=LOG, + header="Attempting to download+extract+upload %s images." % len(locations)) token = self._get_token() - for url in urls: + for uri in locations: try: - name = Image(url, token).install() + name = Image(uri, token).install() if name: LOG.info("Installed image named %r" % (name)) am_installed += 1 except (IOError, tarfile.TarError) as e: - LOG.exception('Installing %r failed due to: %s', url, e) + LOG.exception('Installing %r failed due to: %s', uri, e) return am_installed diff --git a/stack b/stack index c3a6e445..469b13f4 100755 --- a/stack +++ b/stack @@ -118,7 +118,7 @@ def run(args): loaded_rcs = True root_dir = env.get_key(env_rc.INSTALL_ROOT) if not root_dir: - root_dir = utils.joinpths(sh.gethomedir(), 'openstack') + root_dir = sh.joinpths(sh.gethomedir(), 'openstack') root_dir = sh.abspth(root_dir) setup_root(root_dir) From ed32f9ef600fa4ef6dbae626766ba2e40abb4e43 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 29 Mar 2012 16:35:13 -0700 Subject: [PATCH 8/8] Fixed bug in setting sections that aren't there, and added a little upload image tool/util --- devstack/cfg.py | 2 ++ tools/upload-img.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 tools/upload-img.py diff --git a/devstack/cfg.py b/devstack/cfg.py index 49d6d8ac..c2719451 100644 --- a/devstack/cfg.py +++ b/devstack/cfg.py @@ -117,6 +117,8 @@ class StackConfigParser(IgnoreMissingConfigParser): key = cfg_helpers.make_id(section, option) LOG.audit("Setting config value '%s' for param %r" % (value, key)) self.configs_fetched[key] = value + if not self.has_section(section): + self.add_section(section) IgnoreMissingConfigParser.set(self, section, option, value) def _resolve_replacements(self, value): diff --git a/tools/upload-img.py b/tools/upload-img.py new file mode 100644 index 00000000..b2c3bd07 --- /dev/null +++ b/tools/upload-img.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +import os +import sys +from optparse import OptionParser + +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, + 'devstack', + '__init__.py')): + sys.path.insert(0, possible_topdir) + + +from devstack import log as logging +from devstack import utils +from devstack import cfg +from devstack import passwords +from devstack.image import uploader + + + +if __name__ == "__main__": + parser = OptionParser() + parser.add_option("-u", "--uri", + action="append", + dest="uris", + metavar="URI", + help=("uri to attempt to upload to glance")) + (options, args) = parser.parse_args() + uris = options.uris or list() + uri_sep = ",".join(uris) + utils.configure_logging(3) + config = cfg.StackConfigParser() + config.add_section('img') + config.set('img', 'image_urls', uri_sep) + pw_gen = passwords.PasswordGenerator(config) + uploader = uploader.Service(config, pw_gen) + uploader.install()