x/anvil
Code Issues Proposed changes

Fixing up pkg/pip install/uninstall/setup...

Browse Source
This commit is contained in:
Joshua Harlow 2012-03-15 17:08:30 -07:00
parent a20a06edac
commit d2eb62290e
10 changed files with 149 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
conf/distros/ubuntu-oneiric.yaml
View File

@ -5,7 +5,7 @@
name: ubuntu-oneiric
distro_pattern: Ubuntu(.*)oneiric
packager_name: devstack.packaging.apt:AptPackager
packager_name: devstack.distros.oneiric:OneiricAptPackager
commands:

59
devstack/component.py
View File

@ -62,7 +62,6 @@ class ComponentBase(object):
*args,
**kargs):
# Required vars
self.subsystems = subsystems
self.instances = all_instances
@ -107,6 +106,7 @@ class PkgInstallComponent(ComponentBase):
ComponentBase.__init__(self, *args, **kargs)
self.tracewriter = tr.TraceWriter(tr.trace_fn(self.trace_dir,
tr.IN_TRACE))
self.packages = kargs.get('packages', list())
def _get_download_locations(self):
return list()
@ -149,33 +149,32 @@ class PkgInstallComponent(ComponentBase):
def _get_param_map(self, config_fn):
return dict()
def _get_packages(self):
return self.packages
def install(self):
LOG.debug('Preparing to install packages for %s',
self.component_name)
pkgs = self.component_opts.get('packages', [])
pkgs = self._get_packages()
if pkgs:
pkgnames = sorted([p['name'] for p in pkgs])
LOG.info("Installing packages (%s).", ", ".join(pkgnames))
# FIXME: We should only record the packages we actually
# install without error.
#do this before install just incase it craps out half way through
for pkg in pkgs:
self.tracewriter.package_installed(p['name'], pkg)
#now actually install
self.packager.install_batch(pkgs)
pkg_names = set([p['name'] for p in pkgs])
LOG.info("Setting up %s packages (%s)" % (len(pkg_names), ", ".join(pkg_names)))
for p in pkgs:
self.tracewriter.package_installed(p)
self.packager.install(p)
else:
LOG.info('No packages to install for %s',
self.component_name)
return self.trace_dir
def pre_install(self):
pkgs = self.component_opts.get('packages', [])
pkgs = self._get_packages()
if pkgs:
mp = self._get_param_map(None)
self.packager.pre_install(pkgs, mp)
def post_install(self):
pkgs = self.component_opts.get('packages', [])
pkgs = self._get_packages()
if pkgs:
mp = self._get_param_map(None)
self.packager.post_install(pkgs, mp)
@ -250,24 +249,24 @@ class PkgInstallComponent(ComponentBase):
class PythonInstallComponent(PkgInstallComponent):
def __init__(self, *args, **kargs):
PkgInstallComponent.__init__(self, *args, **kargs)
self.pips = kargs.get('pips', list())
def _get_python_directories(self):
py_dirs = dict()
py_dirs[self.component_name] = self.app_dir
return py_dirs
def _get_pips(self):
return self.pips
def _install_pips(self):
pips = dict((p['name'], p)
for p in self.component_opts.get('pips', [])
)
pips = self._get_pips()
if pips:
LOG.info("Setting up %s pips (%s)",
len(pips), ", ".join(pips.keys()))
#do this before install just incase it craps out half way through
for name in pips.keys():
self.tracewriter.pip_installed(name, pips.get(name))
#now install
pip.install(pips, self.distro)
pip_names = set([p['name'] for p in pips])
LOG.info("Setting up %s pips (%s)", len(pip_names), ", ".join(pip_names))
for pip in pips:
self.tracewriter.pip_installed(pip)
pip.install(pip, self.distro)
def _install_python_setups(self):
pydirs = self._get_python_directories()
@ -276,11 +275,8 @@ class PythonInstallComponent(PkgInstallComponent):
len(pydirs), pydirs)
for (name, wkdir) in pydirs.items():
working_dir = wkdir or self.app_dir
#ensure working dir is there
self.tracewriter.dirs_made(*sh.mkdirslist(working_dir))
#do this before write just incase it craps out half way through
self.tracewriter.py_installed(name, working_dir)
#now actually do it
(stdout, stderr) = sh.execute(*PY_INSTALL,
cwd=working_dir,
run_as_root=True)
@ -356,11 +352,11 @@ class PkgUninstallComponent(ComponentBase):
def _uninstall_pkgs(self):
pkgsfull = self.tracereader.packages_installed()
if pkgsfull:
LOG.info("Potentially removing %s packages (%s)",
len(pkgsfull), ", ".join(sorted(pkgsfull.keys())))
LOG.info("Potentially removing %s packages",
len(pkgsfull))
which_removed = self.packager.remove_batch(pkgsfull)
LOG.info("Actually removed %s packages (%s)",
len(which_removed), ", ".join(sorted(which_removed)))
len(which_removed), ", ".join(which_removed))
def _uninstall_touched_files(self):
filestouched = self.tracereader.files_touched()
@ -399,8 +395,9 @@ class PythonUninstallComponent(PkgUninstallComponent):
def _uninstall_pips(self):
pips = self.tracereader.pips_installed()
if pips:
LOG.info("Uninstalling %s pips.", len(pips))
pip.uninstall(pips, self.distro)
names = set([p['name'] for p in pips])
LOG.info("Uninstalling %s python packages (%s)" % (len(names), ", ".join(names)))
pip.uninstall_batch(pips, self.distro)
def _uninstall_python(self):
pylisting = self.tracereader.py_listing()

6
devstack/distro.py
View File

@ -81,6 +81,12 @@ class Distro(object):
def __repr__(self):
return "\"%s\" using packager \"%s\"" % (self.name, self.packager_name)
def get_packages(self, name):
return self.components[name].get('packages', list())
def get_pips(self, name):
return self.components[name].get('pips', list())
def get_command(self, cmd_key, quiet=False):
if not quiet:
return self.commands[cmd_key]

53
devstack/distros/oneiric.py
View File

@ -1,10 +1,31 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
# Copyright (C) 2012 Dreamhost Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Platform-specific logic for Ubunutu Oneiric components.
"""
import tempfile
import time
from devstack.components import db
from devstack import log as logging
from devstack import shell as sh
from devstack import utils
from devstack.packaging import apt
LOG = logging.getLogger(__name__)
@ -23,3 +44,35 @@ class OneiricDBInstaller(db.DBInstaller):
fc = utils.joinlinesep(*new_lines)
with sh.Rooted(True):
sh.write_file('/etc/mysql/my.cnf', fc)
class OneiricAptPackager(apt.AptPackager):
def _pkg_remove_special(self, name, pkginfo):
if name == 'rabbitmq-server':
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600
LOG.info("Handling special remove of %s." % (name))
pkg_full = self._format_pkg(name, pkginfo.get("version"))
cmd = apt.APT_GET + apt.APT_REMOVE + [pkg_full]
self._execute_apt(cmd)
#probably useful to do this
time.sleep(1)
#purge
cmd = apt.APT_GET + apt.APT_PURGE + [pkg_full]
self._execute_apt(cmd)
return True
return False
def _pkg_install_special(self, name, pkginfo):
if name == 'rabbitmq-server':
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600
LOG.info("Handling special install of %s." % (name))
#this seems to be a temporary fix for that bug
with tempfile.TemporaryFile() as f:
pkg_full = self._format_pkg(name, pkginfo.get("version"))
cmd = apt.APT_GET + apt.APT_INSTALL + [pkg_full]
self._execute_apt(cmd, stdout_fh=f, stderr_fh=f)
return True
return False

2
devstack/env_rc.py
View File

@ -261,7 +261,7 @@ class RcReader(object):
with open(fn, 'r') as fh:
contents = fh.read()
except IOError as e:
LOG.warn("Failed extracting rc file [%s] due to [%s]" % (fn, e.message))
LOG.warn("Failed extracting rc file [%s] due to [%s]" % (fn, e))
return extracted_vars
for line in contents.splitlines():
if self._is_comment(line):

26
devstack/packager.py
View File

@ -26,7 +26,7 @@ class Packager(object):
self.distro = distro
self.keep_packages = keep_packages
def install_batch(self, pkgs):
def install(self, pkg):
raise NotImplementedError()
def remove_batch(self, pkgs):
@ -34,21 +34,21 @@ class Packager(object):
return self._remove_batch(pkgs)
return []
def pre_install(self, pkgs, installparams=None):
for packageinfo in pkgs:
preinstallcmds = packageinfo.get(settings.PRE_INSTALL)
if preinstallcmds:
def pre_install(self, pkgs, params=None):
for info in pkgs:
cmds = info.get(settings.PRE_INSTALL)
if cmds:
LOG.info("Running pre-install commands for package %s.",
packageinfo['name'])
utils.execute_template(*preinstallcmds, params=installparams)
info['name'])
utils.execute_template(*cmds, params=params)
def post_install(self, pkgs, installparams=None):
for packageinfo in pkgs:
postinstallcmds = packageinfo.get(settings.POST_INSTALL)
if postinstallcmds and len(postinstallcmds):
def post_install(self, pkgs, params=None):
for info in pkgs:
cmds = info.get(settings.POST_INSTALL)
if cmds:
LOG.info("Running post-install commands for package %s.",
packageinfo['name'])
utils.execute_template(*postinstallcmds, params=installparams)
info['name'])
utils.execute_template(*cmds, params=params)
def _remove_batch(self, pkgs):
raise NotImplementedError()

66
devstack/packaging/apt.py
View File

@ -14,7 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from tempfile import TemporaryFile
import time
from devstack import log as logging
@ -24,20 +23,21 @@ from devstack import shell as sh
LOG = logging.getLogger("devstack.packaging.apt")
#base apt commands
# Base apt commands
APT_GET = ['apt-get']
APT_PURGE = ["purge", "-y"]
APT_REMOVE = ["remove", "-y"]
APT_INSTALL = ["install", "-y"]
APT_AUTOREMOVE = ['autoremove', '-y']
#should we use remove or purge?
# Should we use remove or purge?
APT_DO_REMOVE = APT_PURGE
#make sure its non-interactive
# Make sure its non-interactive
# http://awaseconfigurations.wordpress.com/tag/debian_frontend/
ENV_ADDITIONS = {'DEBIAN_FRONTEND': 'noninteractive'}
#apt separates its pkg names and versions with a equal sign
# Apt separates its pkg names and versions with a equal sign
VERSION_TEMPL = "%s=%s"
@ -60,7 +60,6 @@ class AptPackager(pack.Packager):
**kargs)
def _remove_batch(self, pkgs):
#form the needed commands
cmds = []
which_removed = []
for info in pkgs:
@ -78,53 +77,22 @@ class AptPackager(pack.Packager):
if cmds:
cmd = APT_GET + APT_DO_REMOVE + cmds
self._execute_apt(cmd)
#clean them out (if we did anything)
if which_removed and self.auto_remove:
cmd = APT_GET + APT_AUTOREMOVE
self._execute_apt(cmd)
return which_removed
def install_batch(self, pkgs):
#form the needed commands
cmds = []
for info in pkgs:
name = info['name']
if self._pkg_install_special(name, info):
continue
pkg_full = self._format_pkg(name, info.get("version"))
cmds.append(pkg_full)
#install them
if cmds:
cmd = APT_GET + APT_INSTALL + cmds
self._execute_apt(cmd)
def _pkg_remove_special(self, name, pkginfo):
#TODO: maybe this should be a subclass that handles these differences
if name == 'rabbitmq-server' and self.distro.name == settings.UBUNTU11:
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600
LOG.info("Handling special remove of %s." % (name))
pkg_full = self._format_pkg(name, pkginfo.get("version"))
cmd = APT_GET + APT_REMOVE + [pkg_full]
self._execute_apt(cmd)
#probably useful to do this
time.sleep(1)
#purge
cmd = APT_GET + APT_PURGE + [pkg_full]
self._execute_apt(cmd)
return True
return False
def _pkg_install_special(self, name, pkginfo):
#TODO: maybe this should be a subclass that handles these differences
if name == 'rabbitmq-server' and self.distro.name == settings.UBUNTU11:
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597
#https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600
LOG.info("Handling special install of %s." % (name))
#this seems to be a temporary fix for that bug
with TemporaryFile() as f:
pkg_full = self._format_pkg(name, pkginfo.get("version"))
def install(self, pkg):
name = pkg['name']
if self._pkg_install_special(name, pkg):
return
else:
pkg_full = self._format_pkg(name, pkg.get("version"))
cmd = APT_GET + APT_INSTALL + [pkg_full]
self._execute_apt(cmd, stdout_fh=f, stderr_fh=f)
return True
self._execute_apt(cmd)
def _pkg_remove_special(self, name, info):
return False
def _pkg_install_special(self, name, info):
return False

32
devstack/pip.py
View File

@ -25,21 +25,16 @@ PIP_UNINSTALL_CMD_OPTS = ['-y', '-q']
PIP_INSTALL_CMD_OPTS = ['-q']
def install(pips, distro):
pipnames = sorted(pips.keys())
root_cmd = distro.commands.get('pip', 'pip')
LOG.info("Installing python packages (%s) using command (%s)" % (", ".join(pipnames), root_cmd))
for name in pipnames:
pipfull = name
pipinfo = pips.get(name)
if pipinfo and pipinfo.get('version'):
version = pipinfo.get('version')
def install(pip, distro):
name = pip['name']
root_cmd = distro.get_command('pip')
LOG.audit("Installing python package (%s) using pip command (%s)" % (name, root_cmd))
name_full = name
version = pip.get('version')
if version is not None:
pipfull = pipfull + "==" + str(version)
LOG.debug("Installing python package (%s)" % (pipfull))
real_cmd = [root_cmd, 'install']
real_cmd += PIP_INSTALL_CMD_OPTS
options = pipinfo.get('options')
name_full += "==" + str(version)
real_cmd = [root_cmd, 'install'] + PIP_INSTALL_CMD_OPTS
options = pip.get('options')
if options is not None:
LOG.debug("Using pip options: %s" % (str(options)))
real_cmd += [str(options)]
@ -47,11 +42,10 @@ def install(pips, distro):
sh.execute(*real_cmd, run_as_root=True)
def uninstall(pips, distro, skip_errors=True):
pipnames = sorted(pips.keys())
root_cmd = distro.commands.get('pip', 'pip')
LOG.info("Uninstalling python packages (%s) using command (%s)" % (", ".join(pipnames), root_cmd))
for name in pipnames:
def uninstall_batch(pips, distro, skip_errors=True):
names = [p['name'] for p in pips]
root_cmd = distro.get_command('pip')
for name in names:
try:
LOG.debug("Uninstalling python package (%s)" % (name))
cmd = [root_cmd, 'uninstall'] + PIP_UNINSTALL_CMD_OPTS + [str(name)]

9
devstack/progs/actions.py
View File

@ -145,7 +145,8 @@ class ActionRunner(object):
self.cfg = cfg
self.pw_gen = passwords.PasswordGenerator(self.cfg, kargs.get('prompt_for_passwords', True))
pkg_cls = distro.get_packager_factory()
self.pkg_manager = pkg_cls(self.distro.name, kargs.get('keep_old', False))
self.keep_old = kargs.get('keep_old')
self.pkg_manager = pkg_cls(self.distro, self.keep_old)
self.force = kargs.get('force', False)
self.kargs = kargs
@ -196,8 +197,10 @@ class ActionRunner(object):
cls_kvs['subsystems'] = set(subsystems.get(c, list()))
cls_kvs['all_instances'] = instances
cls_kvs['name'] = c
# FIXME:
cls_kvs['keep_old'] = False
# FIXME: we are always sending these... (even if not used)
cls_kvs['keep_old'] = self.keep_old
cls_kvs['packages'] = self.distro.get_packages(c)
cls_kvs['pips'] = self.distro.get_pips(c)
LOG.debug("Using k/v map %s", cls_kvs)
instances[c] = cls(*list(), **cls_kvs)
return instances

26
devstack/trace.py
View File

@ -95,12 +95,9 @@ class TraceWriter(object):
what['from'] = uri
self.trace(DOWNLOADED, json.dumps(what))
def pip_installed(self, name, pip_info):
def pip_installed(self, pip_info):
self._start()
what = dict()
what['name'] = name
what['pip_meta'] = pip_info
self.trace(PIP_INSTALL, json.dumps(what))
self.trace(PIP_INSTALL, json.dumps(pip_info))
def dirs_made(self, *dirs):
self._start()
@ -111,12 +108,9 @@ class TraceWriter(object):
self._start()
self.trace(FILE_TOUCHED, fn)
def package_installed(self, name, pkg_info):
def package_installed(self, pkg_info):
self._start()
what = dict()
what['name'] = name
what['pkg_meta'] = pkg_info
self.trace(PKG_INSTALL, json.dumps(what))
self.trace(PKG_INSTALL, json.dumps(pkg_info))
def started_info(self, name, info_fn):
self._start()
@ -242,7 +236,7 @@ class TraceReader(object):
def pips_installed(self):
lines = self.read()
pips_installed = dict()
pips_installed = list()
pip_list = list()
for (cmd, action) in lines:
if cmd == PIP_INSTALL and len(action):
@ -250,14 +244,12 @@ class TraceReader(object):
for pip_data in pip_list:
pip_info_full = json.loads(pip_data)
if type(pip_info_full) is dict:
name = pip_info_full.get('name')
if name:
pips_installed[name] = pip_info_full.get('pip_meta')
pips_installed.append(pip_info_full)
return pips_installed
def packages_installed(self):
lines = self.read()
pkgs_installed = dict()
pkgs_installed = list()
pkg_list = list()
for (cmd, action) in lines:
if cmd == PKG_INSTALL and len(action):
@ -265,7 +257,5 @@ class TraceReader(object):
for pkg_data in pkg_list:
pkg_info = json.loads(pkg_data)
if type(pkg_info) is dict:
name = pkg_info.get('name')
if name:
pkgs_installed[name] = pkg_info.get('pkg_meta')
pkgs_installed.append(pkg_info)
return pkgs_installed