From d2eb62290e27dea05024f24daef30df75bd96870 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 15 Mar 2012 17:08:30 -0700 Subject: [PATCH] Fixing up pkg/pip install/uninstall/setup... --- conf/distros/ubuntu-oneiric.yaml | 2 +- devstack/component.py | 59 +++++++++++++++---------------- devstack/distro.py | 6 ++++ devstack/distros/oneiric.py | 53 ++++++++++++++++++++++++++++ devstack/env_rc.py | 2 +- devstack/packager.py | 26 +++++++------- devstack/packaging/apt.py | 60 ++++++++------------------------ devstack/pip.py | 44 ++++++++++------------- devstack/progs/actions.py | 9 +++-- devstack/trace.py | 26 +++++--------- 10 files changed, 149 insertions(+), 138 deletions(-) diff --git a/conf/distros/ubuntu-oneiric.yaml b/conf/distros/ubuntu-oneiric.yaml index f35095a8..cde362c8 100644 --- a/conf/distros/ubuntu-oneiric.yaml +++ b/conf/distros/ubuntu-oneiric.yaml @@ -5,7 +5,7 @@ name: ubuntu-oneiric distro_pattern: Ubuntu(.*)oneiric -packager_name: devstack.packaging.apt:AptPackager +packager_name: devstack.distros.oneiric:OneiricAptPackager commands: diff --git a/devstack/component.py b/devstack/component.py index 2e7809e2..46feaf92 100644 --- a/devstack/component.py +++ b/devstack/component.py @@ -62,7 +62,6 @@ class ComponentBase(object): *args, **kargs): - # Required vars self.subsystems = subsystems self.instances = all_instances @@ -107,6 +106,7 @@ class PkgInstallComponent(ComponentBase): ComponentBase.__init__(self, *args, **kargs) self.tracewriter = tr.TraceWriter(tr.trace_fn(self.trace_dir, tr.IN_TRACE)) + self.packages = kargs.get('packages', list()) def _get_download_locations(self): return list() @@ -149,33 +149,32 @@ class PkgInstallComponent(ComponentBase): def _get_param_map(self, config_fn): return dict() + def _get_packages(self): + return self.packages + def install(self): LOG.debug('Preparing to install packages for %s', self.component_name) - pkgs = self.component_opts.get('packages', []) + pkgs = self._get_packages() if pkgs: - pkgnames = sorted([p['name'] for p in pkgs]) - LOG.info("Installing packages (%s).", ", ".join(pkgnames)) - # FIXME: We should only record the packages we actually - # install without error. - #do this before install just incase it craps out half way through - for pkg in pkgs: - self.tracewriter.package_installed(p['name'], pkg) - #now actually install - self.packager.install_batch(pkgs) + pkg_names = set([p['name'] for p in pkgs]) + LOG.info("Setting up %s packages (%s)" % (len(pkg_names), ", ".join(pkg_names))) + for p in pkgs: + self.tracewriter.package_installed(p) + self.packager.install(p) else: LOG.info('No packages to install for %s', self.component_name) return self.trace_dir def pre_install(self): - pkgs = self.component_opts.get('packages', []) + pkgs = self._get_packages() if pkgs: mp = self._get_param_map(None) self.packager.pre_install(pkgs, mp) def post_install(self): - pkgs = self.component_opts.get('packages', []) + pkgs = self._get_packages() if pkgs: mp = self._get_param_map(None) self.packager.post_install(pkgs, mp) @@ -250,24 +249,24 @@ class PkgInstallComponent(ComponentBase): class PythonInstallComponent(PkgInstallComponent): def __init__(self, *args, **kargs): PkgInstallComponent.__init__(self, *args, **kargs) + self.pips = kargs.get('pips', list()) def _get_python_directories(self): py_dirs = dict() py_dirs[self.component_name] = self.app_dir return py_dirs + + def _get_pips(self): + return self.pips def _install_pips(self): - pips = dict((p['name'], p) - for p in self.component_opts.get('pips', []) - ) + pips = self._get_pips() if pips: - LOG.info("Setting up %s pips (%s)", - len(pips), ", ".join(pips.keys())) - #do this before install just incase it craps out half way through - for name in pips.keys(): - self.tracewriter.pip_installed(name, pips.get(name)) - #now install - pip.install(pips, self.distro) + pip_names = set([p['name'] for p in pips]) + LOG.info("Setting up %s pips (%s)", len(pip_names), ", ".join(pip_names)) + for pip in pips: + self.tracewriter.pip_installed(pip) + pip.install(pip, self.distro) def _install_python_setups(self): pydirs = self._get_python_directories() @@ -276,11 +275,8 @@ class PythonInstallComponent(PkgInstallComponent): len(pydirs), pydirs) for (name, wkdir) in pydirs.items(): working_dir = wkdir or self.app_dir - #ensure working dir is there self.tracewriter.dirs_made(*sh.mkdirslist(working_dir)) - #do this before write just incase it craps out half way through self.tracewriter.py_installed(name, working_dir) - #now actually do it (stdout, stderr) = sh.execute(*PY_INSTALL, cwd=working_dir, run_as_root=True) @@ -356,11 +352,11 @@ class PkgUninstallComponent(ComponentBase): def _uninstall_pkgs(self): pkgsfull = self.tracereader.packages_installed() if pkgsfull: - LOG.info("Potentially removing %s packages (%s)", - len(pkgsfull), ", ".join(sorted(pkgsfull.keys()))) + LOG.info("Potentially removing %s packages", + len(pkgsfull)) which_removed = self.packager.remove_batch(pkgsfull) LOG.info("Actually removed %s packages (%s)", - len(which_removed), ", ".join(sorted(which_removed))) + len(which_removed), ", ".join(which_removed)) def _uninstall_touched_files(self): filestouched = self.tracereader.files_touched() @@ -399,8 +395,9 @@ class PythonUninstallComponent(PkgUninstallComponent): def _uninstall_pips(self): pips = self.tracereader.pips_installed() if pips: - LOG.info("Uninstalling %s pips.", len(pips)) - pip.uninstall(pips, self.distro) + names = set([p['name'] for p in pips]) + LOG.info("Uninstalling %s python packages (%s)" % (len(names), ", ".join(names))) + pip.uninstall_batch(pips, self.distro) def _uninstall_python(self): pylisting = self.tracereader.py_listing() diff --git a/devstack/distro.py b/devstack/distro.py index 80926df7..40c0c335 100644 --- a/devstack/distro.py +++ b/devstack/distro.py @@ -81,6 +81,12 @@ class Distro(object): def __repr__(self): return "\"%s\" using packager \"%s\"" % (self.name, self.packager_name) + def get_packages(self, name): + return self.components[name].get('packages', list()) + + def get_pips(self, name): + return self.components[name].get('pips', list()) + def get_command(self, cmd_key, quiet=False): if not quiet: return self.commands[cmd_key] diff --git a/devstack/distros/oneiric.py b/devstack/distros/oneiric.py index 2f08d56f..34059091 100644 --- a/devstack/distros/oneiric.py +++ b/devstack/distros/oneiric.py @@ -1,10 +1,31 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. +# Copyright (C) 2012 Dreamhost Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + """Platform-specific logic for Ubunutu Oneiric components. """ +import tempfile +import time + from devstack.components import db from devstack import log as logging from devstack import shell as sh from devstack import utils +from devstack.packaging import apt LOG = logging.getLogger(__name__) @@ -23,3 +44,35 @@ class OneiricDBInstaller(db.DBInstaller): fc = utils.joinlinesep(*new_lines) with sh.Rooted(True): sh.write_file('/etc/mysql/my.cnf', fc) + + +class OneiricAptPackager(apt.AptPackager): + + def _pkg_remove_special(self, name, pkginfo): + if name == 'rabbitmq-server': + #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597 + #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600 + LOG.info("Handling special remove of %s." % (name)) + pkg_full = self._format_pkg(name, pkginfo.get("version")) + cmd = apt.APT_GET + apt.APT_REMOVE + [pkg_full] + self._execute_apt(cmd) + #probably useful to do this + time.sleep(1) + #purge + cmd = apt.APT_GET + apt.APT_PURGE + [pkg_full] + self._execute_apt(cmd) + return True + return False + + def _pkg_install_special(self, name, pkginfo): + if name == 'rabbitmq-server': + #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597 + #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600 + LOG.info("Handling special install of %s." % (name)) + #this seems to be a temporary fix for that bug + with tempfile.TemporaryFile() as f: + pkg_full = self._format_pkg(name, pkginfo.get("version")) + cmd = apt.APT_GET + apt.APT_INSTALL + [pkg_full] + self._execute_apt(cmd, stdout_fh=f, stderr_fh=f) + return True + return False diff --git a/devstack/env_rc.py b/devstack/env_rc.py index 3c9e70f4..2717be40 100644 --- a/devstack/env_rc.py +++ b/devstack/env_rc.py @@ -261,7 +261,7 @@ class RcReader(object): with open(fn, 'r') as fh: contents = fh.read() except IOError as e: - LOG.warn("Failed extracting rc file [%s] due to [%s]" % (fn, e.message)) + LOG.warn("Failed extracting rc file [%s] due to [%s]" % (fn, e)) return extracted_vars for line in contents.splitlines(): if self._is_comment(line): diff --git a/devstack/packager.py b/devstack/packager.py index c07f8553..da409535 100644 --- a/devstack/packager.py +++ b/devstack/packager.py @@ -26,7 +26,7 @@ class Packager(object): self.distro = distro self.keep_packages = keep_packages - def install_batch(self, pkgs): + def install(self, pkg): raise NotImplementedError() def remove_batch(self, pkgs): @@ -34,21 +34,21 @@ class Packager(object): return self._remove_batch(pkgs) return [] - def pre_install(self, pkgs, installparams=None): - for packageinfo in pkgs: - preinstallcmds = packageinfo.get(settings.PRE_INSTALL) - if preinstallcmds: + def pre_install(self, pkgs, params=None): + for info in pkgs: + cmds = info.get(settings.PRE_INSTALL) + if cmds: LOG.info("Running pre-install commands for package %s.", - packageinfo['name']) - utils.execute_template(*preinstallcmds, params=installparams) + info['name']) + utils.execute_template(*cmds, params=params) - def post_install(self, pkgs, installparams=None): - for packageinfo in pkgs: - postinstallcmds = packageinfo.get(settings.POST_INSTALL) - if postinstallcmds and len(postinstallcmds): + def post_install(self, pkgs, params=None): + for info in pkgs: + cmds = info.get(settings.POST_INSTALL) + if cmds: LOG.info("Running post-install commands for package %s.", - packageinfo['name']) - utils.execute_template(*postinstallcmds, params=installparams) + info['name']) + utils.execute_template(*cmds, params=params) def _remove_batch(self, pkgs): raise NotImplementedError() diff --git a/devstack/packaging/apt.py b/devstack/packaging/apt.py index c8008471..89e8efd8 100644 --- a/devstack/packaging/apt.py +++ b/devstack/packaging/apt.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -from tempfile import TemporaryFile import time from devstack import log as logging @@ -24,20 +23,21 @@ from devstack import shell as sh LOG = logging.getLogger("devstack.packaging.apt") -#base apt commands +# Base apt commands APT_GET = ['apt-get'] APT_PURGE = ["purge", "-y"] APT_REMOVE = ["remove", "-y"] APT_INSTALL = ["install", "-y"] APT_AUTOREMOVE = ['autoremove', '-y'] -#should we use remove or purge? +# Should we use remove or purge? APT_DO_REMOVE = APT_PURGE -#make sure its non-interactive +# Make sure its non-interactive +# http://awaseconfigurations.wordpress.com/tag/debian_frontend/ ENV_ADDITIONS = {'DEBIAN_FRONTEND': 'noninteractive'} -#apt separates its pkg names and versions with a equal sign +# Apt separates its pkg names and versions with a equal sign VERSION_TEMPL = "%s=%s" @@ -60,7 +60,6 @@ class AptPackager(pack.Packager): **kargs) def _remove_batch(self, pkgs): - #form the needed commands cmds = [] which_removed = [] for info in pkgs: @@ -78,53 +77,22 @@ class AptPackager(pack.Packager): if cmds: cmd = APT_GET + APT_DO_REMOVE + cmds self._execute_apt(cmd) - #clean them out (if we did anything) if which_removed and self.auto_remove: cmd = APT_GET + APT_AUTOREMOVE self._execute_apt(cmd) return which_removed - def install_batch(self, pkgs): - #form the needed commands - cmds = [] - for info in pkgs: - name = info['name'] - if self._pkg_install_special(name, info): - continue - pkg_full = self._format_pkg(name, info.get("version")) - cmds.append(pkg_full) - #install them - if cmds: - cmd = APT_GET + APT_INSTALL + cmds + def install(self, pkg): + name = pkg['name'] + if self._pkg_install_special(name, pkg): + return + else: + pkg_full = self._format_pkg(name, pkg.get("version")) + cmd = APT_GET + APT_INSTALL + [pkg_full] self._execute_apt(cmd) - def _pkg_remove_special(self, name, pkginfo): - #TODO: maybe this should be a subclass that handles these differences - if name == 'rabbitmq-server' and self.distro.name == settings.UBUNTU11: - #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597 - #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600 - LOG.info("Handling special remove of %s." % (name)) - pkg_full = self._format_pkg(name, pkginfo.get("version")) - cmd = APT_GET + APT_REMOVE + [pkg_full] - self._execute_apt(cmd) - #probably useful to do this - time.sleep(1) - #purge - cmd = APT_GET + APT_PURGE + [pkg_full] - self._execute_apt(cmd) - return True + def _pkg_remove_special(self, name, info): return False - def _pkg_install_special(self, name, pkginfo): - #TODO: maybe this should be a subclass that handles these differences - if name == 'rabbitmq-server' and self.distro.name == settings.UBUNTU11: - #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597 - #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600 - LOG.info("Handling special install of %s." % (name)) - #this seems to be a temporary fix for that bug - with TemporaryFile() as f: - pkg_full = self._format_pkg(name, pkginfo.get("version")) - cmd = APT_GET + APT_INSTALL + [pkg_full] - self._execute_apt(cmd, stdout_fh=f, stderr_fh=f) - return True + def _pkg_install_special(self, name, info): return False diff --git a/devstack/pip.py b/devstack/pip.py index 8673f8f4..74ef7893 100644 --- a/devstack/pip.py +++ b/devstack/pip.py @@ -25,33 +25,27 @@ PIP_UNINSTALL_CMD_OPTS = ['-y', '-q'] PIP_INSTALL_CMD_OPTS = ['-q'] -def install(pips, distro): - pipnames = sorted(pips.keys()) - root_cmd = distro.commands.get('pip', 'pip') - LOG.info("Installing python packages (%s) using command (%s)" % (", ".join(pipnames), root_cmd)) - for name in pipnames: - pipfull = name - pipinfo = pips.get(name) - if pipinfo and pipinfo.get('version'): - version = pipinfo.get('version') - if version is not None: - pipfull = pipfull + "==" + str(version) - LOG.debug("Installing python package (%s)" % (pipfull)) - real_cmd = [root_cmd, 'install'] - real_cmd += PIP_INSTALL_CMD_OPTS - options = pipinfo.get('options') - if options is not None: - LOG.debug("Using pip options: %s" % (str(options))) - real_cmd += [str(options)] - real_cmd += [pipfull] - sh.execute(*real_cmd, run_as_root=True) +def install(pip, distro): + name = pip['name'] + root_cmd = distro.get_command('pip') + LOG.audit("Installing python package (%s) using pip command (%s)" % (name, root_cmd)) + name_full = name + version = pip.get('version') + if version is not None: + name_full += "==" + str(version) + real_cmd = [root_cmd, 'install'] + PIP_INSTALL_CMD_OPTS + options = pip.get('options') + if options is not None: + LOG.debug("Using pip options: %s" % (str(options))) + real_cmd += [str(options)] + real_cmd += [pipfull] + sh.execute(*real_cmd, run_as_root=True) -def uninstall(pips, distro, skip_errors=True): - pipnames = sorted(pips.keys()) - root_cmd = distro.commands.get('pip', 'pip') - LOG.info("Uninstalling python packages (%s) using command (%s)" % (", ".join(pipnames), root_cmd)) - for name in pipnames: +def uninstall_batch(pips, distro, skip_errors=True): + names = [p['name'] for p in pips] + root_cmd = distro.get_command('pip') + for name in names: try: LOG.debug("Uninstalling python package (%s)" % (name)) cmd = [root_cmd, 'uninstall'] + PIP_UNINSTALL_CMD_OPTS + [str(name)] diff --git a/devstack/progs/actions.py b/devstack/progs/actions.py index 6019947b..d5842631 100644 --- a/devstack/progs/actions.py +++ b/devstack/progs/actions.py @@ -145,7 +145,8 @@ class ActionRunner(object): self.cfg = cfg self.pw_gen = passwords.PasswordGenerator(self.cfg, kargs.get('prompt_for_passwords', True)) pkg_cls = distro.get_packager_factory() - self.pkg_manager = pkg_cls(self.distro.name, kargs.get('keep_old', False)) + self.keep_old = kargs.get('keep_old') + self.pkg_manager = pkg_cls(self.distro, self.keep_old) self.force = kargs.get('force', False) self.kargs = kargs @@ -196,8 +197,10 @@ class ActionRunner(object): cls_kvs['subsystems'] = set(subsystems.get(c, list())) cls_kvs['all_instances'] = instances cls_kvs['name'] = c - # FIXME: - cls_kvs['keep_old'] = False + # FIXME: we are always sending these... (even if not used) + cls_kvs['keep_old'] = self.keep_old + cls_kvs['packages'] = self.distro.get_packages(c) + cls_kvs['pips'] = self.distro.get_pips(c) LOG.debug("Using k/v map %s", cls_kvs) instances[c] = cls(*list(), **cls_kvs) return instances diff --git a/devstack/trace.py b/devstack/trace.py index 0c18e87f..8b029775 100644 --- a/devstack/trace.py +++ b/devstack/trace.py @@ -95,12 +95,9 @@ class TraceWriter(object): what['from'] = uri self.trace(DOWNLOADED, json.dumps(what)) - def pip_installed(self, name, pip_info): + def pip_installed(self, pip_info): self._start() - what = dict() - what['name'] = name - what['pip_meta'] = pip_info - self.trace(PIP_INSTALL, json.dumps(what)) + self.trace(PIP_INSTALL, json.dumps(pip_info)) def dirs_made(self, *dirs): self._start() @@ -111,12 +108,9 @@ class TraceWriter(object): self._start() self.trace(FILE_TOUCHED, fn) - def package_installed(self, name, pkg_info): + def package_installed(self, pkg_info): self._start() - what = dict() - what['name'] = name - what['pkg_meta'] = pkg_info - self.trace(PKG_INSTALL, json.dumps(what)) + self.trace(PKG_INSTALL, json.dumps(pkg_info)) def started_info(self, name, info_fn): self._start() @@ -242,7 +236,7 @@ class TraceReader(object): def pips_installed(self): lines = self.read() - pips_installed = dict() + pips_installed = list() pip_list = list() for (cmd, action) in lines: if cmd == PIP_INSTALL and len(action): @@ -250,14 +244,12 @@ class TraceReader(object): for pip_data in pip_list: pip_info_full = json.loads(pip_data) if type(pip_info_full) is dict: - name = pip_info_full.get('name') - if name: - pips_installed[name] = pip_info_full.get('pip_meta') + pips_installed.append(pip_info_full) return pips_installed def packages_installed(self): lines = self.read() - pkgs_installed = dict() + pkgs_installed = list() pkg_list = list() for (cmd, action) in lines: if cmd == PKG_INSTALL and len(action): @@ -265,7 +257,5 @@ class TraceReader(object): for pkg_data in pkg_list: pkg_info = json.loads(pkg_data) if type(pkg_info) is dict: - name = pkg_info.get('name') - if name: - pkgs_installed[name] = pkg_info.get('pkg_meta') + pkgs_installed.append(pkg_info) return pkgs_installed