Fixing up pkg/pip install/uninstall/setup...

conf/distros/ubuntu-oneiric.yaml

@@ -5,7 +5,7 @@
 name: ubuntu-oneiric
 distro_pattern: Ubuntu(.*)oneiric
 
-packager_name: devstack.packaging.apt:AptPackager
+packager_name: devstack.distros.oneiric:OneiricAptPackager
 
 commands:
 

devstack/component.py

@@ -62,7 +62,6 @@ class ComponentBase(object):
                  *args,
                  **kargs):
 
-        # Required vars
         self.subsystems = subsystems
         self.instances = all_instances
 
@@ -107,6 +106,7 @@ class PkgInstallComponent(ComponentBase):
         ComponentBase.__init__(self, *args, **kargs)
         self.tracewriter = tr.TraceWriter(tr.trace_fn(self.trace_dir,
                                                       tr.IN_TRACE))
+        self.packages = kargs.get('packages', list())
 
     def _get_download_locations(self):
         return list()
@@ -149,33 +149,32 @@ class PkgInstallComponent(ComponentBase):
     def _get_param_map(self, config_fn):
         return dict()
 
+    def _get_packages(self):
+        return self.packages
+
     def install(self):
         LOG.debug('Preparing to install packages for %s',
                   self.component_name)
-        pkgs = self.component_opts.get('packages', [])
+        pkgs = self._get_packages()
         if pkgs:
-            pkgnames = sorted([p['name'] for p in pkgs])
+            pkg_names = set([p['name'] for p in pkgs])
-            LOG.info("Installing packages (%s).", ", ".join(pkgnames))
+            LOG.info("Setting up %s packages (%s)" % (len(pkg_names), ", ".join(pkg_names)))
-            # FIXME: We should only record the packages we actually
+            for p in pkgs:
-            # install without error.
+                self.tracewriter.package_installed(p)
-            #do this before install just incase it craps out half way through
+                self.packager.install(p)
-            for pkg in pkgs:
-                self.tracewriter.package_installed(p['name'], pkg)
-            #now actually install
-            self.packager.install_batch(pkgs)
         else:
             LOG.info('No packages to install for %s',
                      self.component_name)
         return self.trace_dir
 
     def pre_install(self):
-        pkgs = self.component_opts.get('packages', [])
+        pkgs = self._get_packages()
         if pkgs:
             mp = self._get_param_map(None)
             self.packager.pre_install(pkgs, mp)
 
     def post_install(self):
-        pkgs = self.component_opts.get('packages', [])
+        pkgs = self._get_packages()
         if pkgs:
             mp = self._get_param_map(None)
             self.packager.post_install(pkgs, mp)
@@ -250,24 +249,24 @@ class PkgInstallComponent(ComponentBase):
 class PythonInstallComponent(PkgInstallComponent):
     def __init__(self, *args, **kargs):
         PkgInstallComponent.__init__(self, *args, **kargs)
+        self.pips = kargs.get('pips', list())
 
     def _get_python_directories(self):
         py_dirs = dict()
         py_dirs[self.component_name] = self.app_dir
         return py_dirs
+        
+    def _get_pips(self):
+        return self.pips
 
     def _install_pips(self):
-        pips = dict((p['name'], p)
+        pips = self._get_pips()
-                    for p in self.component_opts.get('pips', [])
-                    )
         if pips:
-            LOG.info("Setting up %s pips (%s)",
+            pip_names = set([p['name'] for p in pips])
-                     len(pips), ", ".join(pips.keys()))
+            LOG.info("Setting up %s pips (%s)", len(pip_names), ", ".join(pip_names))
-            #do this before install just incase it craps out half way through
+            for pip in pips:
-            for name in pips.keys():
+                self.tracewriter.pip_installed(pip)
-                self.tracewriter.pip_installed(name, pips.get(name))
+                pip.install(pip, self.distro)
-            #now install
-            pip.install(pips, self.distro)
 
     def _install_python_setups(self):
         pydirs = self._get_python_directories()
@@ -276,11 +275,8 @@ class PythonInstallComponent(PkgInstallComponent):
                      len(pydirs), pydirs)
             for (name, wkdir) in pydirs.items():
                 working_dir = wkdir or self.app_dir
-                #ensure working dir is there
                 self.tracewriter.dirs_made(*sh.mkdirslist(working_dir))
-                #do this before write just incase it craps out half way through
                 self.tracewriter.py_installed(name, working_dir)
-                #now actually do it
                 (stdout, stderr) = sh.execute(*PY_INSTALL,
                                                cwd=working_dir,
                                                run_as_root=True)
@@ -356,11 +352,11 @@ class PkgUninstallComponent(ComponentBase):
     def _uninstall_pkgs(self):
         pkgsfull = self.tracereader.packages_installed()
         if pkgsfull:
-            LOG.info("Potentially removing %s packages (%s)",
+            LOG.info("Potentially removing %s packages",
-                     len(pkgsfull), ", ".join(sorted(pkgsfull.keys())))
+                     len(pkgsfull))
             which_removed = self.packager.remove_batch(pkgsfull)
             LOG.info("Actually removed %s packages (%s)",
-                     len(which_removed), ", ".join(sorted(which_removed)))
+                     len(which_removed), ", ".join(which_removed))
 
     def _uninstall_touched_files(self):
         filestouched = self.tracereader.files_touched()
@@ -399,8 +395,9 @@ class PythonUninstallComponent(PkgUninstallComponent):
     def _uninstall_pips(self):
         pips = self.tracereader.pips_installed()
         if pips:
-            LOG.info("Uninstalling %s pips.", len(pips))
+            names = set([p['name'] for p in pips])
-            pip.uninstall(pips, self.distro)
+            LOG.info("Uninstalling %s python packages (%s)" % (len(names), ", ".join(names)))
+            pip.uninstall_batch(pips, self.distro)
 
     def _uninstall_python(self):
         pylisting = self.tracereader.py_listing()

devstack/distro.py

@@ -81,6 +81,12 @@ class Distro(object):
     def __repr__(self):
         return "\"%s\" using packager \"%s\"" % (self.name, self.packager_name)
 
+    def get_packages(self, name):
+        return self.components[name].get('packages', list())
+
+    def get_pips(self, name):
+        return self.components[name].get('pips', list())
+
     def get_command(self, cmd_key, quiet=False):
         if not quiet:
             return self.commands[cmd_key]

devstack/distros/oneiric.py

@@ -1,10 +1,31 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#    Copyright (C) 2012 Dreamhost Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
 """Platform-specific logic for Ubunutu Oneiric components.
 """
 
+import tempfile
+import time
+
 from devstack.components import db
 from devstack import log as logging
 from devstack import shell as sh
 from devstack import utils
+from devstack.packaging import apt
 
 LOG = logging.getLogger(__name__)
 
@@ -23,3 +44,35 @@ class OneiricDBInstaller(db.DBInstaller):
         fc = utils.joinlinesep(*new_lines)
         with sh.Rooted(True):
             sh.write_file('/etc/mysql/my.cnf', fc)
+
+
+class OneiricAptPackager(apt.AptPackager):
+
+    def _pkg_remove_special(self, name, pkginfo):
+        if name == 'rabbitmq-server':
+            #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597
+            #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600
+            LOG.info("Handling special remove of %s." % (name))
+            pkg_full = self._format_pkg(name, pkginfo.get("version"))
+            cmd = apt.APT_GET + apt.APT_REMOVE + [pkg_full]
+            self._execute_apt(cmd)
+            #probably useful to do this
+            time.sleep(1)
+            #purge
+            cmd = apt.APT_GET + apt.APT_PURGE + [pkg_full]
+            self._execute_apt(cmd)
+            return True
+        return False
+
+    def _pkg_install_special(self, name, pkginfo):
+        if name == 'rabbitmq-server':
+            #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597
+            #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600
+            LOG.info("Handling special install of %s." % (name))
+            #this seems to be a temporary fix for that bug
+            with tempfile.TemporaryFile() as f:
+                pkg_full = self._format_pkg(name, pkginfo.get("version"))
+                cmd = apt.APT_GET + apt.APT_INSTALL + [pkg_full]
+                self._execute_apt(cmd, stdout_fh=f, stderr_fh=f)
+                return True
+        return False

devstack/env_rc.py

@@ -261,7 +261,7 @@ class RcReader(object):
             with open(fn, 'r') as fh:
                 contents = fh.read()
         except IOError as e:
-            LOG.warn("Failed extracting rc file [%s] due to [%s]" % (fn, e.message))
+            LOG.warn("Failed extracting rc file [%s] due to [%s]" % (fn, e))
             return extracted_vars
         for line in contents.splitlines():
             if self._is_comment(line):

devstack/packager.py

@@ -26,7 +26,7 @@ class Packager(object):
         self.distro = distro
         self.keep_packages = keep_packages
 
-    def install_batch(self, pkgs):
+    def install(self, pkg):
         raise NotImplementedError()
 
     def remove_batch(self, pkgs):
@@ -34,21 +34,21 @@ class Packager(object):
             return self._remove_batch(pkgs)
         return []
 
-    def pre_install(self, pkgs, installparams=None):
+    def pre_install(self, pkgs, params=None):
-        for packageinfo in pkgs:
+        for info in pkgs:
-            preinstallcmds = packageinfo.get(settings.PRE_INSTALL)
+            cmds = info.get(settings.PRE_INSTALL)
-            if preinstallcmds:
+            if cmds:
                 LOG.info("Running pre-install commands for package %s.",
-                         packageinfo['name'])
+                         info['name'])
-                utils.execute_template(*preinstallcmds, params=installparams)
+                utils.execute_template(*cmds, params=params)
 
-    def post_install(self, pkgs, installparams=None):
+    def post_install(self, pkgs, params=None):
-        for packageinfo in pkgs:
+        for info in pkgs:
-            postinstallcmds = packageinfo.get(settings.POST_INSTALL)
+            cmds = info.get(settings.POST_INSTALL)
-            if postinstallcmds and len(postinstallcmds):
+            if cmds:
                 LOG.info("Running post-install commands for package %s.",
-                         packageinfo['name'])
+                         info['name'])
-                utils.execute_template(*postinstallcmds, params=installparams)
+                utils.execute_template(*cmds, params=params)
 
     def _remove_batch(self, pkgs):
         raise NotImplementedError()

devstack/packaging/apt.py

@@ -14,7 +14,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from tempfile import TemporaryFile
 import time
 
 from devstack import log as logging
@@ -24,20 +23,21 @@ from devstack import shell as sh
 
 LOG = logging.getLogger("devstack.packaging.apt")
 
-#base apt commands
+# Base apt commands
 APT_GET = ['apt-get']
 APT_PURGE = ["purge", "-y"]
 APT_REMOVE = ["remove", "-y"]
 APT_INSTALL = ["install", "-y"]
 APT_AUTOREMOVE = ['autoremove', '-y']
 
-#should we use remove or purge?
+# Should we use remove or purge?
 APT_DO_REMOVE = APT_PURGE
 
-#make sure its non-interactive
+# Make sure its non-interactive
+# http://awaseconfigurations.wordpress.com/tag/debian_frontend/
 ENV_ADDITIONS = {'DEBIAN_FRONTEND': 'noninteractive'}
 
-#apt separates its pkg names and versions with a equal sign
+# Apt separates its pkg names and versions with a equal sign
 VERSION_TEMPL = "%s=%s"
 
 
@@ -60,7 +60,6 @@ class AptPackager(pack.Packager):
             **kargs)
 
     def _remove_batch(self, pkgs):
-        #form the needed commands
         cmds = []
         which_removed = []
         for info in pkgs:
@@ -78,53 +77,22 @@ class AptPackager(pack.Packager):
         if cmds:
             cmd = APT_GET + APT_DO_REMOVE + cmds
             self._execute_apt(cmd)
-        #clean them out (if we did anything)
         if which_removed and self.auto_remove:
             cmd = APT_GET + APT_AUTOREMOVE
             self._execute_apt(cmd)
         return which_removed
 
-    def install_batch(self, pkgs):
+    def install(self, pkg):
-        #form the needed commands
+        name = pkg['name']
-        cmds = []
+        if self._pkg_install_special(name, pkg):
-        for info in pkgs:
+            return
-            name = info['name']
+        else:
-            if self._pkg_install_special(name, info):
+            pkg_full = self._format_pkg(name, pkg.get("version")) 
-                continue
+            cmd = APT_GET + APT_INSTALL + [pkg_full]
-            pkg_full = self._format_pkg(name, info.get("version"))
-            cmds.append(pkg_full)
-        #install them
-        if cmds:
-            cmd = APT_GET + APT_INSTALL + cmds
             self._execute_apt(cmd)
 
-    def _pkg_remove_special(self, name, pkginfo):
+    def _pkg_remove_special(self, name, info):
-        #TODO: maybe this should be a subclass that handles these differences
-        if name == 'rabbitmq-server' and self.distro.name == settings.UBUNTU11:
-            #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597
-            #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600
-            LOG.info("Handling special remove of %s." % (name))
-            pkg_full = self._format_pkg(name, pkginfo.get("version"))
-            cmd = APT_GET + APT_REMOVE + [pkg_full]
-            self._execute_apt(cmd)
-            #probably useful to do this
-            time.sleep(1)
-            #purge
-            cmd = APT_GET + APT_PURGE + [pkg_full]
-            self._execute_apt(cmd)
-            return True
         return False
 
-    def _pkg_install_special(self, name, pkginfo):
+    def _pkg_install_special(self, name, info):
-        #TODO: maybe this should be a subclass that handles these differences
-        if name == 'rabbitmq-server' and self.distro.name == settings.UBUNTU11:
-            #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597
-            #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600
-            LOG.info("Handling special install of %s." % (name))
-            #this seems to be a temporary fix for that bug
-            with TemporaryFile() as f:
-                pkg_full = self._format_pkg(name, pkginfo.get("version"))
-                cmd = APT_GET + APT_INSTALL + [pkg_full]
-                self._execute_apt(cmd, stdout_fh=f, stderr_fh=f)
-                return True
         return False

devstack/pip.py

@@ -25,33 +25,27 @@ PIP_UNINSTALL_CMD_OPTS = ['-y', '-q']
 PIP_INSTALL_CMD_OPTS = ['-q']
 
 
-def install(pips, distro):
+def install(pip, distro):
-    pipnames = sorted(pips.keys())
+    name = pip['name']
-    root_cmd = distro.commands.get('pip', 'pip')
+    root_cmd = distro.get_command('pip')
-    LOG.info("Installing python packages (%s) using command (%s)" % (", ".join(pipnames), root_cmd))
+    LOG.audit("Installing python package (%s) using pip command (%s)" % (name, root_cmd))
-    for name in pipnames:
+    name_full = name
-        pipfull = name
+    version = pip.get('version')
-        pipinfo = pips.get(name)
+    if version is not None:
-        if pipinfo and pipinfo.get('version'):
+        name_full += "==" + str(version)
-            version = pipinfo.get('version')
+    real_cmd = [root_cmd, 'install'] + PIP_INSTALL_CMD_OPTS
-            if version is not None:
+    options = pip.get('options')
-                pipfull = pipfull + "==" + str(version)
+    if options is not None:
-        LOG.debug("Installing python package (%s)" % (pipfull))
+        LOG.debug("Using pip options: %s" % (str(options)))
-        real_cmd = [root_cmd, 'install']
+        real_cmd += [str(options)]
-        real_cmd += PIP_INSTALL_CMD_OPTS
+    real_cmd += [pipfull]
-        options = pipinfo.get('options')
+    sh.execute(*real_cmd, run_as_root=True)
-        if options is not None:
-            LOG.debug("Using pip options: %s" % (str(options)))
-            real_cmd += [str(options)]
-        real_cmd += [pipfull]
-        sh.execute(*real_cmd, run_as_root=True)
 
 
-def uninstall(pips, distro, skip_errors=True):
+def uninstall_batch(pips, distro, skip_errors=True):
-    pipnames = sorted(pips.keys())
+    names = [p['name'] for p in pips]
-    root_cmd = distro.commands.get('pip', 'pip')
+    root_cmd = distro.get_command('pip')
-    LOG.info("Uninstalling python packages (%s) using command (%s)" % (", ".join(pipnames), root_cmd))
+    for name in names:
-    for name in pipnames:
         try:
             LOG.debug("Uninstalling python package (%s)" % (name))
             cmd = [root_cmd, 'uninstall'] + PIP_UNINSTALL_CMD_OPTS + [str(name)]

devstack/progs/actions.py

@@ -145,7 +145,8 @@ class ActionRunner(object):
         self.cfg = cfg
         self.pw_gen = passwords.PasswordGenerator(self.cfg, kargs.get('prompt_for_passwords', True))
         pkg_cls = distro.get_packager_factory()
-        self.pkg_manager = pkg_cls(self.distro.name, kargs.get('keep_old', False))
+        self.keep_old = kargs.get('keep_old')
+        self.pkg_manager = pkg_cls(self.distro, self.keep_old)
         self.force = kargs.get('force', False)
         self.kargs = kargs
 
@@ -196,8 +197,10 @@ class ActionRunner(object):
             cls_kvs['subsystems'] = set(subsystems.get(c, list()))
             cls_kvs['all_instances'] = instances
             cls_kvs['name'] = c
-            # FIXME:
+            # FIXME: we are always sending these... (even if not used)
-            cls_kvs['keep_old'] = False
+            cls_kvs['keep_old'] = self.keep_old
+            cls_kvs['packages'] = self.distro.get_packages(c)
+            cls_kvs['pips'] = self.distro.get_pips(c)
             LOG.debug("Using k/v map %s", cls_kvs)
             instances[c] = cls(*list(), **cls_kvs)
         return instances

devstack/trace.py

@@ -95,12 +95,9 @@ class TraceWriter(object):
         what['from'] = uri
         self.trace(DOWNLOADED, json.dumps(what))
 
-    def pip_installed(self, name, pip_info):
+    def pip_installed(self, pip_info):
         self._start()
-        what = dict()
+        self.trace(PIP_INSTALL, json.dumps(pip_info))
-        what['name'] = name
-        what['pip_meta'] = pip_info
-        self.trace(PIP_INSTALL, json.dumps(what))
 
     def dirs_made(self, *dirs):
         self._start()
@@ -111,12 +108,9 @@ class TraceWriter(object):
         self._start()
         self.trace(FILE_TOUCHED, fn)
 
-    def package_installed(self, name, pkg_info):
+    def package_installed(self, pkg_info):
         self._start()
-        what = dict()
+        self.trace(PKG_INSTALL, json.dumps(pkg_info))
-        what['name'] = name
-        what['pkg_meta'] = pkg_info
-        self.trace(PKG_INSTALL, json.dumps(what))
 
     def started_info(self, name, info_fn):
         self._start()
@@ -242,7 +236,7 @@ class TraceReader(object):
 
     def pips_installed(self):
         lines = self.read()
-        pips_installed = dict()
+        pips_installed = list()
         pip_list = list()
         for (cmd, action) in lines:
             if cmd == PIP_INSTALL and len(action):
@@ -250,14 +244,12 @@ class TraceReader(object):
         for pip_data in pip_list:
             pip_info_full = json.loads(pip_data)
             if type(pip_info_full) is dict:
-                name = pip_info_full.get('name')
+                pips_installed.append(pip_info_full)
-                if name:
-                    pips_installed[name] = pip_info_full.get('pip_meta')
         return pips_installed
 
     def packages_installed(self):
         lines = self.read()
-        pkgs_installed = dict()
+        pkgs_installed = list()
         pkg_list = list()
         for (cmd, action) in lines:
             if cmd == PKG_INSTALL and len(action):
@@ -265,7 +257,5 @@ class TraceReader(object):
         for pkg_data in pkg_list:
             pkg_info = json.loads(pkg_data)
             if type(pkg_info) is dict:
-                name = pkg_info.get('name')
+                pkgs_installed.append(pkg_info)
-                if name:
-                    pkgs_installed[name] = pkg_info.get('pkg_meta')
         return pkgs_installed