Merge pull request #172 from harlowja/master

Allow packager to state when a package was already installed, make pip use the common base class and audit adjustments in shell logging. Fixing up to be in sync (again).

conf/distros/fedora-16.yaml

@@ -197,9 +197,6 @@ components:
             running: devstack.components.horizon:HorizonRuntime
             uninstall: devstack.components.horizon:HorizonUninstaller
         packages:
-        -   name: Django
-            removable: true
-            version: 1.3*
         -   name: django-registration
             removable: true
             version: 0.7*
@@ -260,6 +257,9 @@ components:
         -   name: pyxattr
             removable: true
             version: 0.5*
+        pips:
+        -   name: django
+            version: 1.4
     keystone:
         action_classes:
             install: devstack.components.keystone:KeystoneInstaller

conf/distros/rhel-6.yaml

@@ -276,7 +276,7 @@ components:
         -   name: SQLAlchemy
             version: 0.7.5
         -   name: django
-            version: 1.3.1
+            version: 1.4
         -   name: django-mailer
             version: 0.1.0
         -   name: django-nose

conf/distros/ubuntu-oneiric.yaml

@@ -238,9 +238,6 @@ components:
         -   name: python-dateutil
             removable: true
             version: 1.4*
-        -   name: python-django
-            removable: true
-            version: 1.3*
         -   name: python-django-mailer
             removable: true
             version: 0.2*
@@ -275,6 +272,8 @@ components:
             removable: true
             version: 1.0*
         pips:
+        -   name: django
+            version: 1.4
         -   name: django-nose-selenium
             version: 0.7.3
         -   name: pycrypto

devstack/component.py

@@ -17,8 +17,10 @@
 import weakref
 
 from devstack import downloader as down
+from devstack import exceptions as excp
 from devstack import importer
 from devstack import log as logging
+from devstack import packager
 from devstack import pip
 from devstack import settings
 from devstack import shell as sh
@@ -122,33 +124,13 @@ class ComponentBase(object):
         return tr.TraceReader(tr.trace_fn(self.trace_dir, tr.IN_TRACE)).exists()
 
 
-class PackageBasedComponentMixin(object):
+class PkgInstallComponent(ComponentBase):
-    """Mix this into classes that need to manipulate
+    def __init__(self, packager_factory, *args, **kargs):
-    OS-level packages.
-    """
-    PACKAGER_KEY_NAME = 'packager_name'
-
-    def __init__(self):
-        self.default_packager = self.distro.get_default_package_manager()
-
-    def get_packager(self, pkg_info):
-        if self.PACKAGER_KEY_NAME in pkg_info:
-            packager_name = pkg_info[self.PACKAGER_KEY_NAME]
-            LOG.debug('Loading custom package manager %r', packager_name)
-            packager = importer.import_entry_point(packager_name)(self.distro)
-        else:
-            LOG.debug('Using default package manager')
-            packager = self.default_packager
-        return packager
-
-
-class PkgInstallComponent(ComponentBase, PackageBasedComponentMixin):
-    def __init__(self, *args, **kargs):
         ComponentBase.__init__(self, *args, **kargs)
-        PackageBasedComponentMixin.__init__(self)
         self.tracewriter = tr.TraceWriter(tr.trace_fn(self.trace_dir,
                                                       tr.IN_TRACE))
         self.packages = kargs.get('packages', list())
+        self.packager_factory = packager_factory
 
     def _get_download_locations(self):
         return list()
@@ -207,46 +189,32 @@ class PkgInstallComponent(ComponentBase, PackageBasedComponentMixin):
         pkg_list = list(self.packages)
         for name in self.desired_subsystems:
             if name in self.subsystem_info:
-                # Todo handle duplicates/version differences?
+                LOG.debug("Extending package list with packages for subsystem %r", name)
-                LOG.debug(
+                pkg_list.extend(self.subsystem_info[name].get('packages', []))
-                    "Extending package list with packages for subsystem %r",
-                    name)
-                subsystem_pkgs = self.subsystem_info[name].get('packages', [])
-                pkg_list.extend(subsystem_pkgs)
         return pkg_list
 
     def install(self):
         LOG.debug('Preparing to install packages for %r', self.component_name)
         pkgs = self._get_packages()
-        if pkgs:
+        pkg_names = set([p['name'] for p in pkgs])
-            pkg_names = set([p['name'] for p in pkgs])
+        utils.log_iterable(pkg_names, logger=LOG,
-            utils.log_iterable(pkg_names, logger=LOG,
+            header="Setting up %s distribution packages" % (len(pkg_names)))
-                header="Setting up %s distribution packages" % (len(pkg_names)))
+        with utils.progress_bar(INSTALL_TITLE, len(pkgs)) as p_bar:
-            with utils.progress_bar(INSTALL_TITLE, len(pkgs)) as p_bar:
+            for (i, p) in enumerate(pkgs):
-                for (i, p) in enumerate(pkgs):
+                self.tracewriter.package_installed(p)
-                    self.tracewriter.package_installed(p)
+                self.packager_factory.get_packager_for(p).install(p)
-                    packager = self.get_packager(p)
+                p_bar.update(i + 1)
-                    packager.install(p)
-                    p_bar.update(i + 1)
-        else:
-            LOG.info('No packages to install for %r', self.component_name)
         return self.trace_dir
 
     def pre_install(self):
         pkgs = self._get_packages()
-        if pkgs:
+        for p in pkgs:
-            mp = self._get_param_map(None)
+            self.packager_factory.get_packager_for(p).pre_install(p, self._get_param_map(None))
-            for p in pkgs:
-                packager = self.get_packager(p)
-                packager.pre_install(p, mp)
 
     def post_install(self):
         pkgs = self._get_packages()
-        if pkgs:
+        for p in pkgs:
-            mp = self._get_param_map(None)
+            self.packager_factory.get_packager_for(p).post_install(p, self._get_param_map(None))
-            for p in pkgs:
-                packager = self.get_packager(p)
-                packager.post_install(p, mp)
 
     def _get_config_files(self):
         return list()
@@ -313,9 +281,10 @@ class PkgInstallComponent(ComponentBase, PackageBasedComponentMixin):
 
 
 class PythonInstallComponent(PkgInstallComponent):
-    def __init__(self, *args, **kargs):
+    def __init__(self, pip_factory, *args, **kargs):
         PkgInstallComponent.__init__(self, *args, **kargs)
         self.pips = kargs.get('pips', list())
+        self.pip_factory = pip_factory
 
     def _get_python_directories(self):
         py_dirs = dict()
@@ -326,7 +295,6 @@ class PythonInstallComponent(PkgInstallComponent):
         pip_list = list(self.pips)
         for name in self.desired_subsystems:
             if name in self.subsystem_info:
-                # TODO handle duplicates/version differences?
                 LOG.debug("Extending pip list with pips for subsystem %r" % (name))
                 subsystem_pips = self.subsystem_info[name].get('pips', list())
                 pip_list.extend(subsystem_pips)
@@ -341,9 +309,21 @@ class PythonInstallComponent(PkgInstallComponent):
             with utils.progress_bar(INSTALL_TITLE, len(pips)) as p_bar:
                 for (i, p) in enumerate(pips):
                     self.tracewriter.pip_installed(p)
-                    pip.install(p, self.distro)
+                    self.pip_factory.get_packager_for(p).install(p)
                     p_bar.update(i + 1)
 
+    def pre_install(self):
+        PkgInstallComponent.pre_install(self)
+        pips = self._get_pips()
+        for p in pips:
+            self.pip_factory.get_packager_for(p).pre_install(p, self._get_param_map(None))
+
+    def post_install(self):
+        PkgInstallComponent.post_install(self)
+        pips = self._get_pips()
+        for p in pips:
+            self.pip_factory.get_packager_for(p).post_install(p, self._get_param_map(None))
+
     def _install_python_setups(self):
         py_dirs = self._get_python_directories()
         if py_dirs:
@@ -378,13 +358,13 @@ class PythonInstallComponent(PkgInstallComponent):
         return trace_dir
 
 
-class PkgUninstallComponent(ComponentBase, PackageBasedComponentMixin):
+class PkgUninstallComponent(ComponentBase):
-    def __init__(self, *args, **kargs):
+    def __init__(self, packager_factory, *args, **kargs):
         ComponentBase.__init__(self, *args, **kargs)
-        PackageBasedComponentMixin.__init__(self)
         self.tracereader = tr.TraceReader(tr.trace_fn(self.trace_dir,
                                                       tr.IN_TRACE))
-        self.keep_old = kargs.get('keep_old')
+        self.keep_old = kargs.get('keep_old', False)
+        self.packager_factory = packager_factory
 
     def unconfigure(self):
         if not self.keep_old:
@@ -434,8 +414,7 @@ class PkgUninstallComponent(ComponentBase, PackageBasedComponentMixin):
             which_removed = set()
             with utils.progress_bar(UNINSTALL_TITLE, len(pkgs), reverse=True) as p_bar:
                 for (i, p) in enumerate(pkgs):
-                    packager = self.get_packager(p)
+                    if self.packager_factory.get_packager_for(p).remove(p):
-                    if packager.remove(p):
                         which_removed.add(p['name'])
                     p_bar.update(i + 1)
             utils.log_iterable(which_removed, logger=LOG,
@@ -468,8 +447,9 @@ class PkgUninstallComponent(ComponentBase, PackageBasedComponentMixin):
 
 
 class PythonUninstallComponent(PkgUninstallComponent):
-    def __init__(self, *args, **kargs):
+    def __init__(self, pip_factory, *args, **kargs):
         PkgUninstallComponent.__init__(self, *args, **kargs)
+        self.pip_factory = pip_factory
 
     def uninstall(self):
         self._uninstall_python()
@@ -487,7 +467,11 @@ class PythonUninstallComponent(PkgUninstallComponent):
                 header="Uninstalling %s python packages" % (len(pip_names)))
             with utils.progress_bar(UNINSTALL_TITLE, len(pips), reverse=True) as p_bar:
                 for (i, p) in enumerate(pips):
-                    pip.uninstall(p, self.distro)
+                    try:
+                        self.pip_factory.get_packager_for(p).remove(p)
+                    except excp.ProcessExecutionError as e:
+                        # NOTE(harlowja): pip seems to die if a pkg isn't there even in quiet mode
+                        pass
                     p_bar.update(i + 1)
 
     def _uninstall_python(self):
@@ -561,7 +545,7 @@ class ProgramRuntime(ComponentBase):
             # Adjust the program options now that we have real locations
             program_opts = utils.param_replace_list(self._get_app_options(app_name), self._get_param_map(app_name))
             # Start it with the given settings
-            LOG.debug("Starting %r using %r", app_name,  run_type)
+            LOG.debug("Starting %r using %r", app_name, run_type)
             details_fn = instance.start(app_name,
                 app_pth=app_pth, app_dir=app_dir, opts=program_opts)
             LOG.info("Started %r details are in %r", app_name, details_fn)
@@ -580,6 +564,7 @@ class ProgramRuntime(ComponentBase):
                 LOG.debug("Stopping %r using %r", app_name, how)
             except RuntimeError as e:
                 LOG.warn("Could not load class %r which should be used to stop %r: %s", how, app_name, e)
+                continue
             if killcls in killer_instances:
                 killer = killer_instances[killcls]
             else:

devstack/components/horizon.py

@@ -130,9 +130,6 @@ class HorizonInstaller(comp.PythonInstallComponent):
         LOG.info("Fixing up database named %r", DB_NAME)
         db.drop_db(self.cfg, self.pw_gen, self.distro, DB_NAME)
         db.create_db(self.cfg, self.pw_gen, self.distro, DB_NAME, utf8=True)
-        # db.grant_permissions(self.cfg, self.pw_gen, self.distro,
-        #                      self.cfg.getdefaulted('db', 'sql_user', 'root')
-        #                      )
 
     def pre_install(self):
         comp.PythonInstallComponent.pre_install(self)
@@ -146,6 +143,7 @@ class HorizonInstaller(comp.PythonInstallComponent):
         self._setup_db()
         self._sync_db()
         self._setup_blackhole()
+        # Anything to fixup after it was installed??
         self._config_fixups()
 
     def _get_apache_user_group(self):

devstack/distro.py

@@ -77,7 +77,6 @@ class Distro(object):
                 'No platform configuration data for %r (%s)' %
                 (plt, distname))
 
-    @decorators.log_debug
     def __init__(self, name, distro_pattern, packager_name, commands, components):
         self.name = name
         self._distro_pattern = re.compile(distro_pattern, re.IGNORECASE)

devstack/distros/oneiric.py

@@ -63,7 +63,7 @@ class RabbitPackager(apt.AptPackager):
         self._execute_apt(cmd)
         return True
 
-    def install(self, pkg):
+    def _install(self, pkg):
         #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597
         #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600
         name = pkg['name']

devstack/distros/rhel6.py

@@ -33,9 +33,6 @@ from devstack.packaging import yum
 
 LOG = logging.getLogger(__name__)
 
-SOCKET_CONF = "/etc/httpd/conf.d/wsgi-socket-prefix.conf"
-HTTPD_CONF = '/etc/httpd/conf/httpd.conf'
-
 # See: http://wiki.libvirt.org/page/SSHPolicyKitSetup
 # FIXME: take from distro config??
 LIBVIRT_POLICY_FN = "/etc/polkit-1/localauthority/50-local.d/50-libvirt-access.pkla"
@@ -68,24 +65,38 @@ class DBInstaller(db.DBInstaller):
 
 class HorizonInstaller(horizon.HorizonInstaller):
 
-    def _config_fixups(self):
+    def _config_fix_wsgi(self):
-        (user, group) = self._get_apache_user_group()
         # This is recorded so it gets cleaned up during uninstall
-        self.tracewriter.file_touched(SOCKET_CONF)
+        self.tracewriter.file_touched("/etc/httpd/conf.d/wsgi-socket-prefix.conf")
-        LOG.info("Fixing up %r and %r files" % (SOCKET_CONF, HTTPD_CONF))
+        LOG.info("Fixing up %r" % ("/etc/httpd/conf.d/wsgi-socket-prefix.conf"))
+        contents = "WSGISocketPrefix %s" % (sh.joinpths(self.log_dir, "wsgi-socket"))
         with sh.Rooted(True):
-            # Fix the socket prefix to someplace we can use
+            # The name seems to need to come after wsgi.conf (so thats what we are doing)
-            fc = "WSGISocketPrefix %s" % (sh.joinpths(self.log_dir, "wsgi-socket"))
+            sh.write_file("/etc/httpd/conf.d/wsgi-socket-prefix.conf", contents)
-            sh.write_file(SOCKET_CONF, fc)
+
-            # Now adjust the run user and group (of httpd.conf)
+    def _config_fix_httpd(self):
-            new_lines = list()
+        LOG.info("Fixing up %r" % ('/etc/httpd/conf/httpd.conf'))
-            for line in sh.load_file(HTTPD_CONF).splitlines():
+        (user, group) = self._get_apache_user_group()
-                if line.startswith("User "):
+        old_lines = sh.load_file('/etc/httpd/conf/httpd.conf').splitlines()
-                    line = "User %s" % (user)
+        new_lines = list()
-                if line.startswith("Group "):
+        for line in old_lines:
-                    line = "Group %s" % (group)
+            # Directives in the configuration files are case-insensitive,
-                new_lines.append(line)
+            # but arguments to directives are often case sensitive...
-            sh.write_file(HTTPD_CONF, utils.joinlinesep(*new_lines))
+            # NOTE(harlowja): we aren't handling multi-line fixups...
+            if re.match("^\s*User\s+(.*)$", line, re.I):
+                line = "User %s" % (user)
+            if re.match("^\s*Group\s+(.*)$", line, re.I):
+                line = "Group %s" % (group)
+            if re.match("^\s*Listen\s+(.*)$", line, re.I):
+                line = "Listen 0.0.0.0:80"
+            new_lines.append(line)
+        contents = utils.joinlinesep(*new_lines)
+        with sh.Rooted(True):
+            sh.write_file('/etc/httpd/conf/httpd.conf', contents)
+
+    def _config_fixups(self):
+        self._config_fix_wsgi()
+        self._config_fix_httpd()
 
 
 class RabbitRuntime(rabbit.RabbitRuntime):
@@ -159,7 +170,7 @@ class YumPackagerWithRelinks(yum.YumPackager):
                     sh.unlink(tgt)
         return response
 
-    def install(self, pkg):
+    def _install(self, pkg):
         yum.YumPackager.install(self, pkg)
         options = pkg.get('packager_options', {})
         links = options.get('links', [])

devstack/packager.py

@@ -15,45 +15,106 @@
 #    under the License.
 
 import abc
+import collections
 
 from devstack import decorators
+from devstack import importer
 from devstack import log as logging
 from devstack import utils
 
 LOG = logging.getLogger("devstack.packager")
 
 
+class PackageRegistry(object):
+
+    def __init__(self):
+        self.installed = dict()
+        self.removed = dict()
+
+
 class Packager(object):
+
     __meta__ = abc.ABCMeta
 
-    @decorators.log_debug
     def __init__(self, distro):
         self.distro = distro
+        self.registry = PackageRegistry()
 
-    @abc.abstractmethod
     def install(self, pkg):
-        pass
+        name = pkg['name']
-
+        version = pkg.get('version')
-    @abc.abstractmethod
+        if name in self.registry.installed:
-    def _remove(self, pkg):
+            existing_version = self.registry.installed[name]
-        pass
+            if version == existing_version:
+                LOG.debug("Skipping install of %r since it already happened.", name)
+            else:
+                if existing_version is not None:
+                    if utils.versionize(existing_version) < utils.versionize(version):
+                        LOG.warn("A request has come in for a newer version of %r v(%s), when v(%s) was previously installed!", name, version, existing_version)
+                    elif utils.versionize(existing_version) > utils.versionize(version):
+                        LOG.warn("A request has come in for a older version of %r v(%s), when v(%s) was previously installed!", name, version, existing_version)
+                else:
+                    LOG.warn("A request has come in for a different version of %r v(%s), when a unspecified version was previously installed!", name, version)
+        self._install(pkg)
+        LOG.debug("Noting that %r - v(%s) was installed.", name, (version or "??"))
+        self.registry.installed[name] = version
+        if name in self.registry.removed:
+            del(self.registry.removed[name])
 
     def remove(self, pkg):
         removable = pkg.get('removable', True)
         if not removable:
             return False
-        return self._remove(pkg)
+        name = pkg['name']
+        if name in self.registry.removed:
+            LOG.debug("Skipping removal of %r since it already happened.", name)
+        else:
+            self._remove(pkg)
+            LOG.debug("Noting that %r was removed.", name)
+            self.registry.removed[name] = True
+            if name in self.registry.installed:
+                del(self.registry.installed[name])
+        return True
 
     def pre_install(self, pkg, params=None):
         cmds = pkg.get('pre-install')
         if cmds:
-            LOG.info("Running pre-install commands for package %r.",
+            LOG.info("Running pre-install commands for package %r.", pkg['name'])
-                     pkg['name'])
             utils.execute_template(*cmds, params=params)
 
     def post_install(self, pkg, params=None):
         cmds = pkg.get('post-install')
         if cmds:
-            LOG.info("Running post-install commands for package %r.",
+            LOG.info("Running post-install commands for package %r.", pkg['name'])
-                     pkg['name'])
             utils.execute_template(*cmds, params=params)
+
+    @abc.abstractmethod
+    def _remove(self, pkg):
+        pass
+
+    @abc.abstractmethod
+    def _install(self, pkg):
+        pass
+
+
+class PackagerFactory(object):
+
+    PACKAGER_KEY_NAME = 'packager_name'
+
+    def __init__(self, distro, default_packager):
+        self.default_packager = default_packager
+        self.distro = distro
+        self.fetched_packagers = dict()
+
+    def get_packager_for(self, pkg_info):
+        if self.PACKAGER_KEY_NAME in pkg_info:
+            packager_name = pkg_info[self.PACKAGER_KEY_NAME]
+            if packager_name in self.fetched_packagers:
+                packager = self.fetched_packagers[packager_name]
+            else:
+                LOG.debug('Loading custom package manager %r for package %r', packager_name, pkg_info['name'])
+                packager = importer.import_entry_point(packager_name)(self.distro)
+                self.fetched_packagers[packager_name] = packager
+        else:
+            packager = self.default_packager
+        return packager

devstack/packaging/apt.py

@@ -40,6 +40,7 @@ VERSION_TEMPL = "%s=%s"
 
 
 class AptPackager(pack.Packager):
+
     def __init__(self, distro):
         pack.Packager.__init__(self, distro)
         # FIXME: Should this be coming from a setting somewhere?
@@ -67,7 +68,7 @@ class AptPackager(pack.Packager):
             self._execute_apt(APT_AUTOREMOVE)
         return True
 
-    def install(self, pkg):
+    def _install(self, pkg):
         name = pkg['name']
         pkg_full = self._format_pkg_name(name, pkg.get("version"))
         cmd = APT_INSTALL + [pkg_full]

devstack/packaging/yum.py

@@ -32,6 +32,7 @@ VERSION_TEMPL = "%s-%s"
 
 
 class YumPackager(pack.Packager):
+
     def __init__(self, distro):
         pack.Packager.__init__(self, distro)
 
@@ -53,7 +54,7 @@ class YumPackager(pack.Packager):
     def _install_special(self, name, info):
         return False
 
-    def install(self, pkg):
+    def _install(self, pkg):
         name = pkg['name']
         if self._install_special(name, pkg):
             return

devstack/persona.py

@@ -29,7 +29,7 @@ class Persona(object):
     @classmethod
     def load_file(cls, fn):
         persona_fn = sh.abspth(fn)
-        LOG.audit("Loading persona from file [%s]", persona_fn)
+        LOG.audit("Loading persona from file %r", persona_fn)
         cls_kvs = None
         try:
             with open(persona_fn, "r") as fh:
@@ -47,7 +47,6 @@ class Persona(object):
                                 cls, cls_kvs, err)
         return instance
 
-    @decorators.log_debug
     def __init__(self, description,
                        supports,
                        components,

devstack/pip.py

@@ -18,43 +18,39 @@
 from devstack import exceptions as excp
 from devstack import log as logging
 from devstack import shell as sh
+from devstack import utils
+from devstack import packager as pack
 
 LOG = logging.getLogger("devstack.pip")
+
 PIP_UNINSTALL_CMD_OPTS = ['-y', '-q']
 PIP_INSTALL_CMD_OPTS = ['-q']
 
 
-def _make_pip_name(name, version):
+class Packager(pack.Packager):
-    if version is None:
-        return str(name)
-    return "%s==%s" % (name, version)
 
+    def _make_pip_name(self, name, version):
+        if version is None:
+            return "%s" % (name)
+        return "%s==%s" % (name, version)
 
-def install(pip, distro):
+    def _install(self, pip):
-    name = pip['name']
+        name = pip['name']
-    root_cmd = distro.get_command_config('pip')
+        root_cmd = self.distro.get_command_config('pip')
-    LOG.audit("Installing python package %r using pip command %s" % (name, root_cmd))
+        LOG.audit("Installing python package %r using pip command %s" % (name, root_cmd))
-    name_full = _make_pip_name(name, pip.get('version'))
+        name_full = self._make_pip_name(name, pip.get('version'))
-    real_cmd = [root_cmd, 'install'] + PIP_INSTALL_CMD_OPTS
+        real_cmd = [root_cmd] + ['install'] + PIP_INSTALL_CMD_OPTS
-    options = pip.get('options')
+        options = pip.get('options')
-    if options:
+        if options:
-        LOG.debug("Using pip options: %s" % (options))
+            LOG.debug("Using pip options: %s" % (options))
-        real_cmd += [str(options)]
+            real_cmd += [str(options)]
-    real_cmd += [name_full]
+        real_cmd += [name_full]
-    sh.execute(*real_cmd, run_as_root=True)
+        sh.execute(*real_cmd, run_as_root=True)
 
-
+    def _remove(self, pip):
-def uninstall(pip, distro, skip_errors=True):
+        root_cmd = self.distro.get_command('pip')
-    root_cmd = distro.get_command('pip')
-    try:
         # Versions don't seem to matter here...
-        name = _make_pip_name(pip['name'], None)
+        name = self._make_pip_name(pip['name'], None)
         LOG.audit("Uninstalling python package %r using pip command %s" % (name, root_cmd))
-        cmd = [root_cmd, 'uninstall'] + PIP_UNINSTALL_CMD_OPTS + [name]
+        cmd = [root_cmd] + ['uninstall'] + PIP_UNINSTALL_CMD_OPTS + [name]
         sh.execute(*cmd, run_as_root=True)
-    except excp.ProcessExecutionError:
-        if skip_errors:
-            LOG.debug(("Ignoring execution error that occured when uninstalling pip %r!"
-                " (this may be ok if it was uninstalled by a previous component)") % (name))
-        else:
-            raise

devstack/progs/actions.py

@@ -19,10 +19,13 @@ import abc
 from devstack import env_rc
 from devstack import exceptions as excp
 from devstack import log as logging
+from devstack import packager
+from devstack import pip
 from devstack import settings
 from devstack import shell as sh
 from devstack import utils
 
+
 LOG = logging.getLogger("devstack.progs.actions")
 
 
@@ -74,6 +77,8 @@ class ActionRunner(object):
         desired_subsystems = persona.wanted_subsystems or {}
         component_opts = persona.component_options or {}
         instances = {}
+        pip_factory = packager.PackagerFactory(self.distro, pip.Packager(self.distro))
+        pkg_factory = packager.PackagerFactory(self.distro, self.distro.get_default_package_manager())
         for c in components:
             (cls, my_info) = self.distro.extract_component(c, self.NAME)
             LOG.debug("Constructing class %s" % (cls))
@@ -86,6 +91,8 @@ class ActionRunner(object):
             cls_kvs['keep_old'] = self.keep_old
             cls_kvs['desired_subsystems'] = desired_subsystems.get(c, set())
             cls_kvs['options'] = component_opts.get(c, {})
+            cls_kvs['pip_factory'] = pip_factory
+            cls_kvs['packager_factory'] = pkg_factory
             # The above is not overrideable...
             for (k, v) in my_info.items():
                 if k not in cls_kvs:

devstack/shell.py

@@ -145,8 +145,9 @@ def execute(*cmd, **kwargs):
     else:
         process_env = env.get()
 
-    LOG.debug("With environment %s", process_env)
+    # LOG.debug("With environment %s", process_env)
     demoter = None
+
     def demoter_functor(user_uid, user_gid):
         def doit():
             os.setregid(user_gid, user_gid)
@@ -155,9 +156,10 @@ def execute(*cmd, **kwargs):
 
     if not run_as_root:
         (user_uid, user_gid) = get_suids()
-        if user_uid and user_gid:
+        LOG.audit("Running as (user=%s, group=%s)", user_uid, user_gid)
-            LOG.debug("Not running as root, we will run with real & effective gid:uid --> %s:%s", user_gid, user_uid)
+        demoter = demoter_functor(user_uid=user_uid, user_gid=user_gid)
-            demoter = demoter_functor(user_uid=user_uid, user_gid=user_gid)
+    else:
+        LOG.audit("Running as (user=%s, group=%s)", ROOT_USER_UID, ROOT_USER_UID)
 
     rc = None
     result = None
@@ -639,7 +641,7 @@ def root_mode(quiet=True):
             raise excp.StackException(msg)
     else:
         try:
-            LOG.debug("Escalating permissions to (user=%s, group=%s)" % (root_uid, root_gid))
+            LOG.audit("Escalating permissions to (user=%s, group=%s)" % (root_uid, root_gid))
             os.setreuid(0, root_uid)
             os.setregid(0, root_gid)
         except OSError:
@@ -653,7 +655,7 @@ def user_mode(quiet=True):
     (sudo_uid, sudo_gid) = get_suids()
     if sudo_uid is not None and sudo_gid is not None:
         try:
-            LOG.debug("Dropping permissions to (user=%s, group=%s)" % (sudo_uid, sudo_gid))
+            LOG.audit("Dropping permissions to (user=%s, group=%s)" % (sudo_uid, sudo_gid))
             os.setregid(0, sudo_gid)
             os.setreuid(0, sudo_uid)
         except OSError:

devstack/utils.py

@@ -49,8 +49,6 @@ LOG = logging.getLogger("devstack.util")
 DEF_IP = "127.0.0.1"
 IP_LOOKER = '8.8.8.8'
 DEF_IP_VERSION = settings.IPV4
-ALL_NUMS = re.compile(r"^\d+$")
-START_NUMS = re.compile(r"^(\d+)(\D+)")
 STAR_VERSION = 0
 
 # Thx cowsay
@@ -236,31 +234,29 @@ def tempdir():
         sh.deldir(tdir)
 
 
-def versionize(input_version):
+def versionize(input_version, unknown_version="-1.0"):
+    if input_version == None:
+        return distutils.version.LooseVersion(unknown_version)
+    input_version = str(input_version)
     segments = input_version.split(".")
     cleaned_segments = list()
     for piece in segments:
         piece = piece.strip()
         if len(piece) == 0:
-            msg = "Disallowed empty version segment found"
+            cleaned_segments.append("")
-            raise ValueError(msg)
-        piece = piece.strip("*")
-        if len(piece) == 0:
-            cleaned_segments.append(STAR_VERSION)
-        elif ALL_NUMS.match(piece):
-            cleaned_segments.append(int(piece))
         else:
-            piece_match = START_NUMS.match(piece)
+            piece = piece.strip("*")
-            if not piece_match:
+            if len(piece) == 0:
-                msg = "Unknown version identifier %s" % (piece)
+                cleaned_segments.append(STAR_VERSION)
-                raise ValueError(msg)
             else:
-                cleaned_segments.append(int(piece_match.group(1)))
+                try:
+                    piece = int(piece)
+                except ValueError:
+                    pass
+                cleaned_segments.append(piece)
     if not cleaned_segments:
-        msg = "Disallowed empty version found"
+        return distutils.version.LooseVersion(unknown_version)
-        raise ValueError(msg)
+    return distutils.version.LooseVersion(".".join([str(p) for p in cleaned_segments]))
-    num_parts = [str(p) for p in cleaned_segments]
-    return distutils.version.LooseVersion(".".join(num_parts))
 
 
 def sort_versions(versions, descending=True):
@@ -706,8 +702,8 @@ def goodbye(worked):
     print(msg)
 
 
-def welcome(ident):
+def welcome():
-    lower = "| %s %s |" % (ident, version.version_string())
+    lower = "| %s |" % (version.version_string())
     welcome_header = _get_welcome_stack()
     max_line_len = len(max(welcome_header.splitlines(), key=len))
     footer = color_text(settings.PROG_NICE_NAME, 'green')

stack

@@ -107,6 +107,10 @@ def setup_root(root_dir):
 
 
 def run(args):
+
+    (repeat_string, line_max_len) = utils.welcome()
+    print(utils.center_text("Action Runner", repeat_string, line_max_len))
+
     action = args.pop("action", '').strip().lower()
     if action not in actions.get_action_names():
         print(utils.color_text("No valid action specified!", "red"))
@@ -122,9 +126,6 @@ def run(args):
             root_dir = sh.joinpths(sh.gethomedir(), 'openstack')
     root_dir = sh.abspth(root_dir)
     setup_root(root_dir)
-    if not sh.isuseable(root_dir):
-        print(utils.color_text("Unreadable/writeable/executable root directory %r provided!" % (root_dir), "red"))
-        return False
 
     persona_fn = args.pop('persona_fn')
     if not persona_fn or not sh.isfile(persona_fn):
@@ -132,10 +133,6 @@ def run(args):
         return False
     persona_fn = sh.abspth(persona_fn)
 
-    # Welcome!
-    (repeat_string, line_max_len) = utils.welcome(action.upper())
-    print(utils.center_text("Action Runner", repeat_string, line_max_len))
-
     # !!
     # Here on out we should be using the logger (and not print)!!
     # !!