From 2a30054978cf64fa5936a3ad08381b754f8e5667 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 16 Mar 2012 16:51:10 -0700 Subject: [PATCH] More fixups for rhel6 + some ubuntu class name cleanups --- conf/distros/rhel-6.yaml | 270 ++++++++++++++++++++++++------- conf/distros/ubuntu-oneiric.yaml | 2 +- devstack/distros/oneiric.py | 16 +- devstack/distros/rhel6.py | 44 ++++- devstack/packaging/apt.py | 31 ++-- devstack/packaging/yum.py | 72 ++------- devstack/persona.py | 4 +- stack | 6 +- tools/validate_yaml.py | 6 +- tools/yamlpretty.py | 3 +- 10 files changed, 306 insertions(+), 148 deletions(-) diff --git a/conf/distros/rhel-6.yaml b/conf/distros/rhel-6.yaml index 43e91a5d..d0ce1f50 100644 --- a/conf/distros/rhel-6.yaml +++ b/conf/distros/rhel-6.yaml @@ -1,8 +1,8 @@ --- # RedHat Enterprise Linux 6 -distro_pattern: redhat-6\\.2 +distro_pattern: redhat-6(.*) name: rhel-6 -packager_name: devstack.packaging.yum:YumPackager +packager_name: devstack.distros.rhel6:YumPackager commands: apache: restart: @@ -10,7 +10,7 @@ commands: - httpd - restart settings: - # Ensure runs after wsgi.conf (naming wise) + # Ensure runs after wsgi.conf (file naming wise) conf-link-target: /etc/httpd/conf.d/wsgi-horizon-000-default.conf start: - service @@ -83,6 +83,7 @@ components: stop: devstack.components.db:DBRuntime uninstall: devstack.components.db:DBUninstaller general: + install: devstack.components.pkglist:Installer packages: - name: coreutils removable: false @@ -181,7 +182,18 @@ components: - name: wget removable: false version: 1.12* + pips: + - name: coverage + version: 3.5.1 + - name: mock + version: 0.8.0 + - name: nose + version: 1.1.2 + start: devstack.component:EmptyRuntime + stop: devstack.component:EmptyRuntime + uninstall: devstack.components.pkglist:Uninstaller glance: + install: devstack.components.glance:GlanceInstaller packages: - name: MySQL-python removable: true @@ -219,7 +231,23 @@ components: name: pyxattr removable: true version: 0.5* + pips: + - name: PasteDeploy + version: 1.5.0 + - name: Routes + version: 1.12.3 + - name: SQLAlchemy + version: 0.7.5 + - name: iso8601 + version: 0.1.4 + - name: pycrypto + options: --upgrade + version: '2.5' + start: devstack.components.glance:GlanceRuntime + stop: devstack.components.glance:GlanceRuntime + uninstall: devstack.components.glance:GlanceUninstaller horizon: + install: devstack.distros.rhel6:HorizonInstaller packages: - name: httpd removable: true @@ -245,7 +273,43 @@ components: name: python-webob1.0 removable: true version: 1.0* + pips: + - name: CherryPy + version: 3.2.2 + - name: Paste + version: 1.7.5.1 + - name: PasteDeploy + version: '1.5' + - name: PasteScript + version: 1.7.5 + - name: Routes + version: 1.12.3 + - name: SQLAlchemy + version: 0.7.5 + - name: django + version: 1.3.1 + - name: django-mailer + version: 0.1.0 + - name: django-nose + version: 0.1.3 + - name: django-nose-selenium + version: 0.7.3 + - name: django-registration + version: '0.7' + - name: django-staticfiles + version: 1.1.2 + - name: pycrypto + options: --upgrade + version: '2.5' + - name: python-cloudfiles + version: 1.7.9.3 + - name: sqlalchemy-migrate + version: 0.7.2 + start: devstack.components.horizon:HorizonRuntime + stop: devstack.components.horizon:HorizonRuntime + uninstall: devstack.components.horizon:HorizonUninstaller keystone: + install: devstack.components.keystone:KeystoneInstaller packages: - name: MySQL-python removable: true @@ -300,7 +364,29 @@ components: - name: sqlite removable: false version: 3.6* + pips: + - name: Paste + version: 1.7.5.1 + - name: PasteDeploy + version: '1.5' + - name: PasteScript + version: 1.7.5 + - name: Routes + version: 1.12.3 + - name: SQLAlchemy + version: 0.7.5 + - name: py-bcrypt + version: '0.2' + - name: pycrypto + options: --upgrade + version: '2.5' + - name: sqlalchemy-migrate + version: 0.7.2 + start: devstack.components.keystone:KeystoneRuntime + stop: devstack.components.keystone:KeystoneRuntime + uninstall: devstack.components.keystone:KeystoneUninstaller keystone-client: + install: devstack.components.keystone_client:KeyStoneClientInstaller packages: - meta: epel: true @@ -312,59 +398,24 @@ components: name: python-prettytable removable: true version: 0.5* + start: devstack.components.keystone_client:KeyStoneClientRuntime + stop: devstack.components.keystone_client:KeyStoneClientRuntime + uninstall: devstack.components.keystone_client:KeyStoneClientUninstaller melange: - packages: null - n-api: - packages: - - name: python-dateutil - removable: false - version: 1.4* - n-cpu: - packages: - - name: iscsi-initiator-utils - removable: true - version: 6.2* - - name: libguestfs - removable: true - version: 1.2.7* - - name: libguestfs-mount - removable: true - version: 1.7.17* - - name: libguestfs-tools - removable: true - version: 1.7.17* - - name: libvirt - removable: true - version: 0.9.4* - - name: libvirt-client - removable: true - version: 0.9.4* - - name: libvirt-python - removable: true - version: 0.9.4* - - name: lvm2 - removable: true - version: 2.02* - - name: qemu-img - removable: true - version: 0.12* - - name: qemu-kvm - removable: true - version: 2* - n-vnc: - packages: null - n-vol: - packages: - - name: iscsi-initiator-utils - removable: true - version: 6.2* - - name: lvm2 - removable: true - version: 2.02* - - name: scsi-target-utils - removable: true - version: 1.0* + install: devstack.components.melange:MelangeInstaller + start: devstack.components.melange:MelangeRuntime + stop: devstack.components.melange:MelangeRuntime + uninstall: devstack.components.melange:MelangeUninstaller + no-vnc: + install: devstack.components.novnc:NoVNCInstaller + pips: + - name: numpy + version: '1.5' + start: devstack.components.novnc:NoVNCRuntime + stop: devstack.components.novnc:NoVNCRuntime + uninstall: devstack.components.novnc:NoVNCUninstaller nova: + install: devstack.components.nova:NovaInstaller packages: - name: MySQL-python removable: true @@ -458,7 +509,81 @@ components: - name: sqlite removable: false version: 3.6* + pips: + - name: Paste + version: 1.7.5.1 + - name: PasteDeploy + version: '1.5' + - name: PasteScript + version: 1.7.5 + - name: Routes + version: 1.12.3 + - name: SQLAlchemy + version: 0.7.5 + - name: Tempita + options: --upgrade + version: 0.5dev + - name: iso8601 + version: 0.1.4 + - name: pycrypto + options: --upgrade + version: '2.5' + - name: sqlalchemy-migrate + version: 0.7.2 + start: devstack.components.nova:NovaRuntime + stop: devstack.components.nova:NovaRuntime + subsystems: + api: + packages: + - name: python-dateutil + removable: false + version: 1.4* + cpu: + packages: + - name: iscsi-initiator-utils + removable: true + version: 6.2* + - name: libguestfs + removable: true + version: 1.2.7* + - name: libguestfs-mount + removable: true + version: 1.7.17* + - name: libguestfs-tools + removable: true + version: 1.7.17* + - name: libvirt + removable: true + version: 0.9.4* + - name: libvirt-client + removable: true + version: 0.9.4* + - name: libvirt-python + removable: true + version: 0.9.4* + - name: lvm2 + removable: true + version: 2.02* + - name: qemu-img + removable: true + version: 0.12* + - name: qemu-kvm + removable: true + version: 2* + vol: + packages: + - name: iscsi-initiator-utils + removable: true + version: 6.2* + - name: lvm2 + removable: true + version: 2.02* + - name: scsi-target-utils + removable: true + version: 1.0* + uninstall: devstack.components.nova:NovaUninstaller nova-client: + install: devstack.components.nova_client:NovaClientInstaller packages: - meta: epel: true @@ -470,7 +595,11 @@ components: name: python-prettytable removable: true version: 0.5* + start: devstack.components.nova_client:NovaClientRuntime + stop: devstack.components.nova_client:NovaClientRuntime + uninstall: devstack.components.nova_client:NovaClientUninstaller quantum: + install: devstack.components.quantum:QuantumInstaller packages: - name: libxml2-python removable: false @@ -490,16 +619,35 @@ components: name: python-greenlet removable: true version: 0.3.1* + pips: + - name: Paste + version: 1.7.5.1 + - name: PasteDeploy + version: '1.5' + - name: PasteScript + version: 1.7.5 + - name: Routes + version: 1.12.3 + - name: SQLAlchemy + version: 0.7.5 + - name: sqlalchemy-migrate + version: 0.7.2 + start: devstack.components.quantum:QuantumRuntime + stop: devstack.components.quantum:QuantumRuntime + uninstall: devstack.components.quantum:QuantumUninstaller quantum-client: + install: devstack.components.quantum_client:QuantumClientInstaller packages: - meta: epel: true name: python-gflags removable: true version: 1.4* - quantum-openvswitch: - packages: null - rabbitmq: + start: devstack.components.quantum_client:QuantumClientRuntime + stop: devstack.components.quantum_client:QuantumClientRuntime + uninstall: devstack.components.quantum_client:QuantumClientUninstaller + rabbit-mq: + install: devstack.components.rabbit:RabbitInstaller packages: - meta: epel: true @@ -519,7 +667,13 @@ components: run_as_root: true removable: true version: 2.6* + start: devstack.components.rabbit:RabbitRuntime + stop: devstack.components.rabbit:RabbitRuntime + uninstall: devstack.components.rabbit:RabbitUninstaller swift: - packages: null + install: devstack.components.swift:SwiftInstaller + start: devstack.components.swift:SwiftRuntime + stop: devstack.components.swift:SwiftRuntime + uninstall: devstack.components.swift:SwiftUninstaller ... diff --git a/conf/distros/ubuntu-oneiric.yaml b/conf/distros/ubuntu-oneiric.yaml index ed6455f6..f67ab36f 100644 --- a/conf/distros/ubuntu-oneiric.yaml +++ b/conf/distros/ubuntu-oneiric.yaml @@ -3,7 +3,7 @@ name: ubuntu-oneiric distro_pattern: Ubuntu(.*)oneiric -packager_name: devstack.distros.oneiric:OneiricAptPackager +packager_name: devstack.distros.oneiric:AptPackager commands: diff --git a/devstack/distros/oneiric.py b/devstack/distros/oneiric.py index 8209b54a..88ca1623 100644 --- a/devstack/distros/oneiric.py +++ b/devstack/distros/oneiric.py @@ -46,33 +46,33 @@ class DBInstaller(db.DBInstaller): sh.write_file('/etc/mysql/my.cnf', fc) -class OneiricAptPackager(apt.AptPackager): +class AptPackager(apt.AptPackager): - def _pkg_remove_special(self, name, pkginfo): + def _remove_special(self, name, info): if name == 'rabbitmq-server': #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597 #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600 LOG.info("Handling special remove of %s." % (name)) - pkg_full = self._format_pkg(name, pkginfo.get("version")) - cmd = apt.APT_GET + apt.APT_REMOVE + [pkg_full] + pkg_full = self._format_pkg_name(name, info.get("version")) + cmd = apt.APT_REMOVE + [pkg_full] self._execute_apt(cmd) #probably useful to do this time.sleep(1) #purge - cmd = apt.APT_GET + apt.APT_PURGE + [pkg_full] + cmd = apt.APT_PURGE + [pkg_full] self._execute_apt(cmd) return True return False - def _pkg_install_special(self, name, pkginfo): + def _install_special(self, name, info): if name == 'rabbitmq-server': #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878597 #https://bugs.launchpad.net/ubuntu/+source/rabbitmq-server/+bug/878600 LOG.info("Handling special install of %s." % (name)) #this seems to be a temporary fix for that bug with tempfile.TemporaryFile() as f: - pkg_full = self._format_pkg(name, pkginfo.get("version")) - cmd = apt.APT_GET + apt.APT_INSTALL + [pkg_full] + pkg_full = self._format_pkg_name(name, info.get("version")) + cmd = apt.APT_INSTALL + [pkg_full] self._execute_apt(cmd, stdout_fh=f, stderr_fh=f) return True return False diff --git a/devstack/distros/rhel6.py b/devstack/distros/rhel6.py index 1338d2da..86546cc8 100644 --- a/devstack/distros/rhel6.py +++ b/devstack/distros/rhel6.py @@ -25,11 +25,25 @@ from devstack import utils from devstack.components import db from devstack.components import horizon +from devstack.packaging import yum + LOG = logging.getLogger(__name__) SOCKET_CONF = "/etc/httpd/conf.d/wsgi-socket-prefix.conf" HTTPD_CONF = '/etc/httpd/conf/httpd.conf' +# Need to relink for rhel (not a bug!) +RHEL_RELINKS = { + 'python-webob1.0': { + "src": '/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/', + 'tgt': '/usr/lib/python2.6/site-packages/webob', + }, + 'python-nose1.1': { + "src": '/usr/lib/python2.6/site-packages/nose-1.1.2-py2.6.egg/nose/', + 'tgt': '/usr/lib/python2.6/site-packages/nose', + }, +} + class DBInstaller(db.DBInstaller): @@ -47,12 +61,12 @@ class DBInstaller(db.DBInstaller): sh.write_file('/etc/my.cnf', fc) -class Rhel6HorizonInstaller(horizon.HorizonInstaller): +class HorizonInstaller(horizon.HorizonInstaller): def _config_fixups(self): (user, group) = self._get_apache_user_group() + # This is recorded so it gets cleaned up during uninstall self.tracewriter.file_touched(SOCKET_CONF) - # Not recorded since we aren't really creating this LOG.info("Fixing up %s and %s files" % (SOCKET_CONF, HTTPD_CONF)) with sh.Rooted(True): # Fix the socket prefix to someplace we can use @@ -67,3 +81,29 @@ class Rhel6HorizonInstaller(horizon.HorizonInstaller): line = "Group %s" % (group) new_lines.append(line) sh.write_file(HTTPD_CONF, utils.joinlinesep(*new_lines)) + + +class YumPackager(yum.YumPackager): + + def _remove_special(self, name, info): + if name in RHEL_RELINKS: + # Note: we don't return true here so that + # the normal package cleanup happens... + sh.unlink(RHEL_RELINKS.get(name).get("tgt")) + return False + + def _install_special(self, name, info): + if name in RHEL_RELINKS: + full_pkg_name = self._format_pkg_name(name, info.get("version")) + install_cmd = yum.YUM_INSTALL + [full_pkg_name] + self._execute_yum(install_cmd) + tgt = RHEL_RELINKS.get(pkgname).get("tgt") + src = RHEL_RELINKS.get(pkgname).get("src") + if not sh.islink(tgt): + # This is actually a feature, EPEL must not conflict with RHEL, so X pkg installs newer version in parallel. + # + # This of course doesn't work when running from git like devstack does.... + sh.symlink(src, tgt) + return True + else: + return False diff --git a/devstack/packaging/apt.py b/devstack/packaging/apt.py index 6bb2054b..7522540a 100644 --- a/devstack/packaging/apt.py +++ b/devstack/packaging/apt.py @@ -44,15 +44,15 @@ class AptPackager(pack.Packager): pack.Packager.__init__(self, distro, keep_packages) self.auto_remove = True - def _format_pkg(self, name, version): + def _format_pkg_name(self, name, version): if version: - pkg_full_name = VERSION_TEMPL % (name, version) + return VERSION_TEMPL % (name, version) else: - pkg_full_name = name - return pkg_full_name + return name def _execute_apt(self, cmd, **kargs): - return sh.execute(*cmd, run_as_root=True, + full_cmd = APT_GET + cmd + return sh.execute(*full_cmd, run_as_root=True, check_exit_code=True, env_overrides=ENV_ADDITIONS, **kargs) @@ -65,32 +65,33 @@ class AptPackager(pack.Packager): removable = info.get('removable', True) if not removable: continue - if self._pkg_remove_special(name, info): + if self._remove_special(name, info): which_removed.append(name) continue - pkg_full = self._format_pkg(name, info.get("version")) + pkg_full = self._format_pkg_name(name, info.get("version")) if pkg_full: cmds.append(pkg_full) which_removed.append(name) if cmds: - cmd = APT_GET + APT_DO_REMOVE + cmds + cmd = APT_DO_REMOVE + cmds self._execute_apt(cmd) if which_removed and self.auto_remove: - cmd = APT_GET + APT_AUTOREMOVE + cmd = APT_AUTOREMOVE self._execute_apt(cmd) return which_removed def install(self, pkg): name = pkg['name'] - if self._pkg_install_special(name, pkg): + if self._install_special(name, pkg): return else: - pkg_full = self._format_pkg(name, pkg.get("version")) - cmd = APT_GET + APT_INSTALL + [pkg_full] - self._execute_apt(cmd) + pkg_full = self._format_pkg_name(name, pkg.get("version")) + if pkg_full: + cmd = APT_INSTALL + [pkg_full] + self._execute_apt(cmd) - def _pkg_remove_special(self, name, info): + def _remove_special(self, name, info): return False - def _pkg_install_special(self, name, info): + def _install_special(self, name, info): return False diff --git a/devstack/packaging/yum.py b/devstack/packaging/yum.py index c71ce617..6148fa1d 100644 --- a/devstack/packaging/yum.py +++ b/devstack/packaging/yum.py @@ -30,74 +30,36 @@ YUM_REMOVE = ['erase', '-y', "-t"] # Yum separates its pkg names and versions with a dash VERSION_TEMPL = "%s-%s" -# Need to relink for rhel (not a bug!) -# TODO: maybe this should be a subclass that handles these differences -RHEL_RELINKS = { - 'python-webob1.0': { - "src": '/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/', - 'tgt': '/usr/lib/python2.6/site-packages/webob', - }, - 'python-nose1.1': { - "src": '/usr/lib/python2.6/site-packages/nose-1.1.2-py2.6.egg/nose/', - 'tgt': '/usr/lib/python2.6/site-packages/nose', - }, -} - class YumPackager(pack.Packager): def __init__(self, distro, keep_packages): pack.Packager.__init__(self, distro, keep_packages) def _format_pkg_name(self, name, version): - if version is not None and len(version): + if version: return VERSION_TEMPL % (name, version) else: return name def _execute_yum(self, cmd, **kargs): - return sh.execute(*cmd, run_as_root=True, + full_cmd = YUM_CMD + cmd + return sh.execute(*full_cmd, run_as_root=True, check_exit_code=True, **kargs) - def _remove_special(self, pkgname, pkginfo): - # TODO: maybe this should be a subclass that handles these differences - # if self.distro.name == settings.RHEL6 and pkgname in RHEL_RELINKS: - # #we don't return true here so that - # #the normal package cleanup happens - # sh.unlink(RHEL_RELINKS.get(pkgname).get("tgt")) + def _remove_special(self, name, info): return False - # TODO: maybe this should be a subclass that handles these differences - def _install_rhel_relinks(self, pkgname, pkginfo): - full_pkg_name = self._format_pkg_name(pkgname, pkginfo.get("version")) - install_cmd = YUM_CMD + YUM_INSTALL + [full_pkg_name] - self._execute_yum(install_cmd) - tgt = RHEL_RELINKS.get(pkgname).get("tgt") - src = RHEL_RELINKS.get(pkgname).get("src") - if not sh.islink(tgt): - # This is actually a feature, EPEL must not conflict with RHEL, so X pkg installs newer version in parallel. - # - # This of course doesn't work when running from git like devstack does.... - sh.symlink(src, tgt) - return True - - # TODO: maybe this should be a subclass that handles these differences - def _install_special(self, pkgname, pkginfo): - # FIXME: - # if self.distro.name == settings.RHEL6 and pkgname in RHEL_RELINKS: - # return self._install_rhel_relinks(pkgname, pkginfo) + def _install_special(self, name, info): return False - def install_batch(self, pkgs): - pkg_full_names = [] - for info in pkgs: - name = info['name'] - if self._install_special(name, info): - continue - full_pkg_name = self._format_pkg_name(name, info.get("version")) - pkg_full_names.append(full_pkg_name) - if pkg_full_names: - cmd = YUM_CMD + YUM_INSTALL + pkg_full_names + def install(self, pkg): + name = pkg['name'] + if self._install_special(name, pkg): + return + else: + full_pkg_name = self._format_pkg_name(name, pkg.get("version")) + cmd = YUM_INSTALL + [full_pkg_name] self._execute_yum(cmd) def _remove_batch(self, pkgs): @@ -111,11 +73,11 @@ class YumPackager(pack.Packager): continue if self._remove_special(name, info): which_removed.append(name) - continue - full_pkg_name = self._format_pkg_name(name, info.get("version")) - pkg_full_names.append(full_pkg_name) - which_removed.append(name) + else: + full_pkg_name = self._format_pkg_name(name, info.get("version")) + pkg_full_names.append(full_pkg_name) + which_removed.append(name) if pkg_full_names: - cmd = YUM_CMD + YUM_REMOVE + pkg_full_names + cmd = YUM_REMOVE + pkg_full_names self._execute_yum(cmd) return which_removed diff --git a/devstack/persona.py b/devstack/persona.py index 9c2041fb..066c2931 100644 --- a/devstack/persona.py +++ b/devstack/persona.py @@ -79,9 +79,9 @@ class Persona(object): # Some sanity checks against the given distro d_name = distro.name if d_name not in self.distro_support: - msg = "Distro %s not supported" % (d_name) + msg = "Distro %r not supported" % (d_name) raise excp.ConfigException(msg) for c in self.wanted_components: if not distro.known_component(c): - raise RuntimeError("Distro %s does not support component %s" % + raise RuntimeError("Distro %r does not support component %r" % (d_name, c)) diff --git a/stack b/stack index 8ded902d..a9a37cc1 100755 --- a/stack +++ b/stack @@ -166,9 +166,9 @@ def run(args): pkg_manager, **args) - LOG.info("Starting action [%s] on %s for distro: %s" % (action, date.rcf8222date(), dist.name)) - LOG.info("Using persona: %s" % (persona_inst)) - LOG.info("In root directory: %s" % (root_dir)) + LOG.info("Starting action %r on %s for distro: %r" % (action, date.rcf8222date(), dist.name)) + LOG.info("Using persona: %r" % (persona_fn)) + LOG.info("In root directory: %r" % (root_dir)) start_time = time.time() runner.run(persona_inst, root_dir) diff --git a/tools/validate_yaml.py b/tools/validate_yaml.py index b925c75c..75432398 100755 --- a/tools/validate_yaml.py +++ b/tools/validate_yaml.py @@ -7,6 +7,6 @@ import sys import yaml - -with open(sys.argv[1], 'r') as f: - yaml.load(f) +if __name__ == "__main__": + with open(sys.argv[1], 'r') as f: + yaml.load(f) diff --git a/tools/yamlpretty.py b/tools/yamlpretty.py index da4e5fbd..1355e9be 100755 --- a/tools/yamlpretty.py +++ b/tools/yamlpretty.py @@ -1,9 +1,10 @@ #!/usr/bin/env python -import yaml import os import sys +import yaml + # See: http://pyyaml.org/wiki/PyYAMLDocumentation if __name__ == "__main__":