From 1df3eae3faa832d1fc419015544688487b0f1fbc Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 31 Jan 2012 16:07:40 -0800 Subject: [PATCH] Updates for rhel6 work. --- conf/pips/glance.json | 15 ++++ conf/pips/horizon.json | 15 ++-- conf/pips/keystone.json | 17 +++- conf/pkgs/db.json | 2 +- conf/pkgs/general.json | 11 ++- conf/pkgs/glance.json | 58 +++++++++++- conf/pkgs/keystone.json | 92 +++++++++++++++++-- devstack/component.py | 4 +- devstack/components/db.py | 155 +++++++++++++++++--------------- devstack/components/glance.py | 13 ++- devstack/components/keystone.py | 2 +- devstack/image/creator.py | 10 ++- devstack/packaging/yum.py | 24 +++++ devstack/pip.py | 30 ++++--- devstack/progs/actions.py | 24 ++--- devstack/settings.py | 8 +- devstack/shell.py | 39 ++++---- devstack/utils.py | 15 ++-- 18 files changed, 378 insertions(+), 156 deletions(-) create mode 100644 conf/pips/glance.json diff --git a/conf/pips/glance.json b/conf/pips/glance.json new file mode 100644 index 00000000..f9dbc2c9 --- /dev/null +++ b/conf/pips/glance.json @@ -0,0 +1,15 @@ +# This is a extended json package definition file +# We allow simple comments (lines starting with a hash symbol) +{ + "ubuntu-oneiric": { + }, + "rhel-6": { + "SQLAlchemy": { + "version": "0.7.5" + }, + #the base is 2.0, need to upgrade + "pycrypto": { + "version": "2.3" + } + } +} diff --git a/conf/pips/horizon.json b/conf/pips/horizon.json index edc2a81c..d787a383 100644 --- a/conf/pips/horizon.json +++ b/conf/pips/horizon.json @@ -3,15 +3,16 @@ { "ubuntu-oneiric": { "django-nose-selenium": { - "version": "0.7.3", - #this seems broke on uninstall so skip errors on uinstall - "skip_uninstall_errors": true + "version": "0.7.3" }, "pycrypto": { - "version": "2.3", - #this seems broke on uninstall so skip errors on uinstall - "skip_uninstall_errors": true + "version": "2.3" } }, - "rhel-6": {} + "rhel-6": { + #the base is 2.0, need to upgrade + "pycrypto": { + "version": "2.3" + } + } } diff --git a/conf/pips/keystone.json b/conf/pips/keystone.json index 4fc5b266..ac1a1e5f 100644 --- a/conf/pips/keystone.json +++ b/conf/pips/keystone.json @@ -3,10 +3,19 @@ { "ubuntu-oneiric": { "passlib": { - "version": "1.5.3", - #this seems broke on uninstall so skip errors on uinstall - "skip_uninstall_errors": true + "version": "1.5.3" } }, - "rhel-6": {} + "rhel-6": { + "SQLAlchemy": { + "version": "0.7.5" + }, + "sqlalchemy-migrate": { + "version": "0.7.2" + }, + #the base is 2.0, need to upgrade + "pycrypto": { + "version": "2.3" + } + } } diff --git a/conf/pkgs/db.json b/conf/pkgs/db.json index 96108956..6dbeb937 100644 --- a/conf/pkgs/db.json +++ b/conf/pkgs/db.json @@ -22,7 +22,7 @@ "removable": true, "pre-install": [ { - # This apparently is a action needed for ubuntu/debian to not prompt for passwords... + # This apparently is a action needed for ubuntu/debian to set the password to something known.... "run_as_root": true, "cmd": [ "debconf-set-selections" diff --git a/conf/pkgs/general.json b/conf/pkgs/general.json index 03b4ea32..4a3a1d3f 100644 --- a/conf/pkgs/general.json +++ b/conf/pkgs/general.json @@ -196,14 +196,19 @@ "version": "0.6.10*", "removable": false }, + # Requires EPEL + "python-distutils-extra": { + "version": "2.29*", + "removable": false + }, "python-dev": { "version": "2.6*", "removable": false }, # For testing - "python-nose": { - # This version seems way old - "version": "0.10*", + # Requires EPEL + "python-nose1.1": { + "version": "1.1*", "removable": true }, # For testing diff --git a/conf/pkgs/glance.json b/conf/pkgs/glance.json index 64f0d937..2daa4b27 100644 --- a/conf/pkgs/glance.json +++ b/conf/pkgs/glance.json @@ -21,6 +21,10 @@ "version": "1.1*", "removable": true }, + "python-prettytable": { + "version": "0.5*", + "removable": true + }, "python-mysqldb": { "version": "1.2*", "removable": true @@ -37,12 +41,64 @@ "version": "0.6*", "removable": true }, + "python-dateutil": { + "version": "1.4*", + "removable": true + }, "python-httplib2": { "version": "0.7.1*", "removable": true } }, "rhel-6": { - # TBD + # Requires EPEL + "python-eventlet": { + "version": "0.9*", + "removable": true + }, + "python-routes": { + "version": "1.10*", + "removable": true + }, + # Requires EPEL + "python-greenlet": { + "version": "0.3.1*", + "removable": true + }, + # Requires EPEL + "python-argparse": { + "version": "1.2*", + "removable": true + }, + "MySQL-python": { + "version": "1.2*", + "removable": true + }, + "python-dateutil": { + "version": "1.4*", + "removable": true + }, + "python-pastedeploy": { + # This might be way to old :( + "version": "1.3.3*", + "removable": true + }, + # Requires EPEL + "python-prettytable": { + "version": "0.5*", + "removable": true + }, + # Requires EPEL + "python-xattr": { + # This might be way to old :( + "version": "0.5*", + "removable": true + }, + # Requires EPEL + "python-httplib2": { + # This might be way to old :( + "version": "0.4*", + "removable": true + } } } diff --git a/conf/pkgs/keystone.json b/conf/pkgs/keystone.json index e1ad53e6..60b75ed9 100644 --- a/conf/pkgs/keystone.json +++ b/conf/pkgs/keystone.json @@ -55,12 +55,14 @@ }, "python-greenlet": { "version": "0.3*", - "allowed": ">=", + "removable": true + }, + "python-argparse": { + "version": "1.1*", "removable": true }, "python-routes": { "version": "1.12*", - "allowed": ">=", "removable": true }, "libldap2-dev": { @@ -71,13 +73,89 @@ "libsasl2-dev": { "version": "2.1*", "removable": true - }, - "python-migrate": { - "version": "0.7*", - "removable": true } }, "rhel-6": { - # TBD + # Requires EPEL + "python-eventlet": { + "version": "0.9*", + "removable": true + }, + "python-lxml": { + # Trashes IPA client, which is probably bad + "version": "2.2*", + "removable": true + }, + "python-pastescript": { + "version": "1.7*", + "removable": true + }, + # Requires EPEL + "python-prettytable": { + "version": "0.5*", + "removable": true + }, + "MySQL-python": { + "version": "1.2*", + "removable": true + }, + "python-paste-deploy": { + # This might be way to old :( + "version": "1.3.3*", + "removable": true + }, + "python-paste": { + "version": "1.7.4*", + "removable": true + }, + "sqlite": { + "version": "3.6*", + # Trashes alot of the base os (so we don't allow it to be removed) + "removable": false + }, + # Requires EPEL + "python-sqlite2": { + # This might be way to old :( + "version": "2.3*", + "removable": true + }, + # Requires EPEL + "python-webob1.0": { + "version": "1.0*", + "removable": true + }, + # Requires EPEL + "python-argparse": { + "version": "1.2*", + "removable": true + }, + # Requires EPEL + "python-greenlet": { + "version": "0.3.1*", + "removable": true + }, + "python-routes": { + "version": "1.10*", + "removable": true + }, + # Requires EPEL + "python-passlib": { + "version": "1.5*", + "removable": true + }, + # Is this right?? + # This is for libldap2 + "openldap": { + # Trashes alot of the base os (so we don't allow it to be removed) + "removable": false, + "version": "2.4*" + }, + # Is this right?? + # This is for libsasl2 + "cyrus-sasl-lib": { + # Trashes alot of the base os (so we don't allow it to be removed) + "version": "2.1*", + "removable": false + } } } diff --git a/devstack/component.py b/devstack/component.py index 13936431..ce6af27e 100644 --- a/devstack/component.py +++ b/devstack/component.py @@ -181,7 +181,7 @@ class PythonInstallComponent(PkgInstallComponent): pips = self._get_pips() if pips: LOG.info("Setting up %s pips (%s)" % (len(pips), ", ".join(pips.keys()))) - pip.install(pips) + pip.install(pips, self.distro) for name in pips.keys(): self.tracewriter.pip_install(name, pips.get(name)) @@ -293,7 +293,7 @@ class PythonUninstallComponent(PkgUninstallComponent): pips = self.tracereader.pips_installed() if pips: LOG.info("Uninstalling %s pips" % (len(pips))) - pip.uninstall(pips) + pip.uninstall(pips, self.distro) def _uninstall_python(self): pylisting = self.tracereader.py_listing() diff --git a/devstack/components/db.py b/devstack/components/db.py index 19890a0b..e9f2628d 100644 --- a/devstack/components/db.py +++ b/devstack/components/db.py @@ -22,6 +22,8 @@ from devstack import shell as sh from devstack import trace as tr from devstack import utils +import time + LOG = logging.getLogger("devstack.components.db") #id @@ -29,6 +31,7 @@ TYPE = settings.DB #used for special setups MYSQL = 'mysql' +START_WAIT_TIME = 10 DB_ACTIONS = { MYSQL: { # Of course these aren't distro independent... @@ -37,37 +40,37 @@ DB_ACTIONS = { 'start': ["service", "mysql", 'start'], 'stop': ["service", 'mysql', "stop"], 'status': ["service", 'mysql', "status"], - 'restart': ["service", 'mysql', "status"], + 'restart': ["service", 'mysql', "restart"], }, settings.RHEL6: { 'start': ["service", "mysqld", 'start'], 'stop': ["service", 'mysqld', "stop"], 'status': ["service", 'mysqld', "status"], - 'restart': ["service", 'mysqld', "status"], + 'restart': ["service", 'mysqld', "restart"], }, }, - # - 'setpwd': ['mysqladmin', '--user=%USER%', 'password', '%NEW_PASSWORD%', - '--password=%PASSWORD%'], + #modification commands + 'set_pwd': ['mysql', '-u', '%USER%', '--password=%OLD_PASSWORD%', '-e', ("\"USE mysql; UPDATE user SET " + " password=PASSWORD('%NEW_PASSWORD%') WHERE User='%USER%'; FLUSH privileges;\"")], 'create_db': ['mysql', '--user=%USER%', '--password=%PASSWORD%', '-e', 'CREATE DATABASE %DB%;'], 'drop_db': ['mysql', '--user=%USER%', '--password=%PASSWORD%', '-e', 'DROP DATABASE IF EXISTS %DB%;'], - 'grant_all': [ - "mysql", - "--user=%USER%", - "--password=%PASSWORD%", - ("-e \"GRANT ALL PRIVILEGES ON *.* TO '%USER%'@'%' " - "identified by '%PASSWORD%';\""), - ], - # we could do this in python directly, but executing allows us to - # not have to sudo the whole program - 'host_adjust': ['perl', '-p', '-i', '-e', "'s/127.0.0.1/0.0.0.0/g'", - '/etc/mysql/my.cnf'], + 'grant_all': ["mysql", "--user=%USER%", "--password=%PASSWORD%", + ("-e \"GRANT ALL PRIVILEGES ON *.* TO '%USER%'@'%' " + "identified by '%PASSWORD%'; flush privileges;\"")], }, } +#annoying adjustments +RHEL_FIX_GRANTS = ['perl', '-p', '-i', '-e', "'s/^skip-grant-tables/#skip-grant-tables/g'", '/etc/my.cnf'] +UBUNTU_HOST_ADJUST = ['perl', '-p', '-i', '-e', "'s/127.0.0.1/0.0.0.0/g'", '/etc/mysql/my.cnf'] +#need to reset pw (this is the prompt) +RESET_PW = "Please enter your current mysql password for user \"%s\" so we can reset it for next time (blank allowed): " +RESET_BASE_PW = '' + +#links about how to reset if it fails SQL_RESET_PW_LINKS = ['https://help.ubuntu.com/community/MysqlPasswordReset', 'http://crashmag.net/resetting-the-root-password-for-mysql-running-on-rhel-or-centos'] #used as a generic error message @@ -88,38 +91,28 @@ class DBUninstaller(comp.PkgUninstallComponent): def pre_uninstall(self): dbtype = self.cfg.get("db", "type") dbactions = DB_ACTIONS.get(dbtype) - - try: - self.runtime.start() - except IOError: - LOG.warn("Could not start your database.") - - # set pwd try: if dbactions and dbtype == MYSQL: LOG.info(("Attempting to reset your mysql password so" " that we can set it the next time you install.")) - pwd_cmd = dbactions.get('setpwd') + pwd_cmd = dbactions.get('set_pwd') if pwd_cmd: + LOG.info("Ensuring your database is started before we operate on it.") + self.runtime.restart() + user = self.cfg.get("db", "sql_user") + pw_prompt = RESET_PW % (user) + old_pw = sh.prompt_password(pw_prompt) params = { - 'PASSWORD': self.cfg.get("passwords", "sql"), - 'USER': self.cfg.get("db", "sql_user"), - 'NEW_PASSWORD': '' + 'OLD_PASSWORD': old_pw, + 'NEW_PASSWORD': RESET_BASE_PW, + 'USER': user, } - cmds = [{ - 'cmd': pwd_cmd, - 'run_as_root': True, - }] - utils.execute_template(*cmds, params=params) + cmds = [{'cmd': pwd_cmd}] + utils.execute_template(*cmds, params=params, shell=True) except IOError: - LOG.warn(("Could not reset mysql password. You might have to manually " - "reset mysql before the next install")) - LOG.info("To aid in this check out: %s", " or ".join(SQL_RESET_PW_LINKS)) - - try: - self.runtime.stop() - except IOError: - LOG.warn("Could not stop your database.") + LOG.warn(("Could not reset the database password. You might have to manually " + "reset the password to \"%s\" before the next install") % (RESET_BASE_PW), exc_info=True) + LOG.info("To aid in this check out: [%s]", " or ".join(SQL_RESET_PW_LINKS)) class DBInstaller(comp.PkgInstallComponent): @@ -140,6 +133,15 @@ class DBInstaller(comp.PkgInstallComponent): } return out + def _configure_db_confs(self): + dbtype = self.cfg.get("db", "type") + if self.distro == settings.RHEL6 and dbtype == MYSQL: + LOG.info("Fixing up rhel 6 mysql configs") + sh.execute(*RHEL_FIX_GRANTS, run_as_root=True) + elif self.distro == settings.UBUNTU11 and dbtype == MYSQL: + LOG.info("Fixing up ubuntu 11 mysql configs") + sh.execute(*UBUNTU_HOST_ADJUST, run_as_root=True) + def _get_pkgs(self): pkgs = comp.PkgInstallComponent._get_pkgs(self) for fn in REQ_PKGS: @@ -150,53 +152,53 @@ class DBInstaller(comp.PkgInstallComponent): def post_install(self): parent_result = comp.PkgInstallComponent.post_install(self) + #fix up the db configs + self._configure_db_confs() + #extra actions to ensure we are granted access dbtype = self.cfg.get("db", "type") dbactions = DB_ACTIONS.get(dbtype) - self.runtime.start() - # set pwd + #set your password try: if dbactions and dbtype == MYSQL: - LOG.info(("Attempting to set your mysql password " - " just incase it wasn't set previously")) - pwd_cmd = dbactions.get('setpwd') + LOG.info(("Attempting to set your mysql password" + " just incase it wasn't set previously.")) + pwd_cmd = dbactions.get('set_pwd') if pwd_cmd: + LOG.info("Ensuring mysql is started.") + self.runtime.restart() params = { 'NEW_PASSWORD': self.cfg.get("passwords", "sql"), - 'PASSWORD': '', - 'USER': self.cfg.get("db", "sql_user") + 'USER': self.cfg.get("db", "sql_user"), + 'OLD_PASSWORD': RESET_BASE_PW, } - cmds = [{ - 'cmd': pwd_cmd, - 'run_as_root': True, - }] - utils.execute_template(*cmds, params=params) + cmds = [{'cmd': pwd_cmd}] + utils.execute_template(*cmds, params=params, shell=True) except IOError: LOG.warn(("Couldn't set your password. It might have already been " - "set by a previous process.")) + "set by a previous process."), exc_info=True) - if dbactions and dbactions.get('grant_all'): - #update the DB to give user 'USER'@'%' full control of the all databases: + #ensure access granted + if dbactions: grant_cmd = dbactions.get('grant_all') - params = self._get_param_map(None) - cmds = list() - cmds.append({ - 'cmd': grant_cmd, - 'run_as_root': False, - }) - #shell seems to be needed here - #since python escapes this to much... - utils.execute_template(*cmds, params=params, shell=True) + if grant_cmd: + user = self.cfg.get("db", "sql_user") + LOG.info("Updating the DB to give user '%s' full control of all databases." % (user)) + LOG.info("Ensuring your database is started.") + self.runtime.restart() + params = { + 'PASSWORD': self.cfg.get("passwords", "sql"), + 'USER': user, + } + cmds = list() + cmds.append({ + 'cmd': grant_cmd, + }) + #shell seems to be needed here + #since python escapes this to much... + utils.execute_template(*cmds, params=params, shell=True) - #special mysql actions - if dbactions and dbtype == MYSQL: - cmd = dbactions.get('host_adjust') - if cmd: - sh.execute(*cmd, run_as_root=True, shell=True) - - #restart it to make sure all good - self.runtime.restart() return parent_result @@ -225,6 +227,8 @@ class DBRuntime(comp.EmptyRuntime): if self.status() == comp.STATUS_STOPPED: startcmd = self._get_run_actions('start', excp.StartException) sh.execute(*startcmd, run_as_root=True) + LOG.info("Please wait %s seconds while it comes up" % START_WAIT_TIME) + time.sleep(START_WAIT_TIME) return 1 else: return 0 @@ -240,14 +244,17 @@ class DBRuntime(comp.EmptyRuntime): def restart(self): restartcmd = self._get_run_actions('restart', excp.RestartException) sh.execute(*restartcmd, run_as_root=True) + #this seems needed? + LOG.info("Please wait %s seconds while it comes up" % START_WAIT_TIME) + time.sleep(START_WAIT_TIME) return 1 def status(self): statuscmd = self._get_run_actions('status', excp.StatusException) (sysout, _) = sh.execute(*statuscmd, check_exit_code=False) - if sysout.find("start/running") != -1: + if sysout.find("running") != -1: return comp.STATUS_STARTED - elif sysout.find("stop/waiting") != -1: + elif sysout.find("stop") != -1: return comp.STATUS_STOPPED else: return comp.STATUS_UNKNOWN diff --git a/devstack/components/glance.py b/devstack/components/glance.py index 5a40f143..2cb4c646 100644 --- a/devstack/components/glance.py +++ b/devstack/components/glance.py @@ -51,6 +51,7 @@ DB_NAME = "glance" #special subcomponents/options that are used in starting to know that images should be uploaded NO_IMG_START = "no-image-upload" +WAIT_ONLINE_TO = 5 #what to start APP_OPTIONS = { @@ -71,6 +72,9 @@ BIN_DIR = 'bin' #the pkg json files glance requires for installation REQ_PKGS = ['general.json', 'glance.json'] +#pip files that glance requires +REQ_PIPS = ['glance.json'] + class GlanceUninstaller(comp.PythonUninstallComponent): def __init__(self, *args, **kargs): @@ -97,6 +101,13 @@ class GlanceInstaller(comp.PythonInstallComponent): #these are the config files we will be adjusting return list(CONFIGS) + def _get_pips(self): + pips = comp.PythonInstallComponent._get_pips(self) + for fn in REQ_PIPS: + full_name = sh.joinpths(settings.STACK_PIP_DIR, fn) + pips = utils.extract_pip_list([full_name], self.distro, pips) + return pips + def _get_pkgs(self): pkgs = comp.PythonInstallComponent._get_pkgs(self) for fn in REQ_PKGS: @@ -205,7 +216,7 @@ class GlanceRuntime(comp.PythonRuntime): if NO_IMG_START not in self.component_opts: #install any images that need activating... # TODO: make this less cheesy - need to wait till glance goes online - time.sleep(1) + time.sleep(WAIT_ONLINE_TO) creator.ImageCreationService(self.cfg).install() diff --git a/devstack/components/keystone.py b/devstack/components/keystone.py index 892ca9f3..5fe19466 100644 --- a/devstack/components/keystone.py +++ b/devstack/components/keystone.py @@ -59,7 +59,7 @@ APP_OPTIONS = { #the pkg json files keystone requires for installation REQ_PKGS = ['general.json', 'keystone.json'] -#pip files that horizon requires +#pip files that keystone requires REQ_PIPS = ['keystone.json'] diff --git a/devstack/image/creator.py b/devstack/image/creator.py index d07f487d..1123b5ad 100644 --- a/devstack/image/creator.py +++ b/devstack/image/creator.py @@ -108,7 +108,9 @@ class Image(object): params = {'TOKEN': self.token, 'IMAGE_NAME': self.image_name} cmd = {'cmd': Image.KERNEL_FORMAT} with open(self.kernel) as file_: - res = utils.execute_template(cmd, params=params, stdin_fh=file_) + res = utils.execute_template(cmd, + params=params, stdin_fh=file_, + close_stdin=True) self.kernel_id = res[0][0].split(':')[1].strip() if self.initrd: @@ -116,7 +118,8 @@ class Image(object): params = {'TOKEN': self.token, 'IMAGE_NAME': self.image_name} cmd = {'cmd': Image.INITRD_FORMAT} with open(self.initrd) as file_: - res = utils.execute_template(cmd, params=params, stdin_fh=file_) + res = utils.execute_template(cmd, params=params, + stdin_fh=file_, close_stdin=True) self.initrd_id = res[0][0].split(':')[1].strip() LOG.info('Adding image %s to glance', self.image_name) @@ -124,7 +127,8 @@ class Image(object): 'KERNEL_ID': self.kernel_id, 'INITRD_ID': self.initrd_id} cmd = {'cmd': Image.IMAGE_FORMAT} with open(self.image) as file_: - utils.execute_template(cmd, params=params, stdin_fh=file_) + utils.execute_template(cmd, params=params, + stdin_fh=file_, close_stdin=True) def _cleanup(self): if self.tmp_folder: diff --git a/devstack/packaging/yum.py b/devstack/packaging/yum.py index 9aef20d7..544c2d71 100644 --- a/devstack/packaging/yum.py +++ b/devstack/packaging/yum.py @@ -16,6 +16,7 @@ from devstack import log as logging from devstack import packager as pack +from devstack import settings from devstack import shell as sh LOG = logging.getLogger("devstack.packaging.yum") @@ -47,9 +48,32 @@ class YumPackager(pack.Packager): **kargs) def _remove_special(self, pkgname, pkginfo): + if pkgname == 'python-webob1.0' and self.distro == settings.RHEL6: + self._remove_webob_rhel() return False + def _remove_webob_rhel(self): + if sh.isdir('/usr/lib/python2.6/site-packages/webob'): + #remove the link we made + rm_cmd = ['rm', '/usr/lib/python2.6/site-packages/webob'] + sh.execute(*rm_cmd, run_as_root=True) + + def _install_webob_rhel(self, pkgname, pkginfo): + full_pkg_name = self._format_pkg_name(pkgname, pkginfo.get("version")) + install_cmd = YUM_CMD + YUM_INSTALL + [full_pkg_name] + self._execute_yum(install_cmd) + #need to fix its link... + if not sh.isdir('/usr/lib/python2.6/site-packages/webob'): + #TODO: this needs to be a bug against that epel pkg + link_cmd = ['ln', '-s', + '/usr/lib/python2.6/site-packages/WebOb-1.0.8-py2.6.egg/webob/', + '/usr/lib/python2.6/site-packages/webob'] + sh.execute(*link_cmd, run_as_root=True) + return True + def _install_special(self, pkgname, pkginfo): + if pkgname == 'python-webob1.0' and self.distro == settings.RHEL6: + return self._install_webob_rhel(pkgname, pkginfo) return False def install_batch(self, pkgs): diff --git a/devstack/pip.py b/devstack/pip.py index 22921633..cafd1ebb 100644 --- a/devstack/pip.py +++ b/devstack/pip.py @@ -18,16 +18,24 @@ from devstack import exceptions as excp from devstack import log as logging from devstack import shell as sh +from devstack import settings LOG = logging.getLogger("devstack.pip") -INSTALL_CMD = ['pip', 'install'] -UNINSTALL_CMD = ['pip', 'uninstall'] + +PIP_UNINSTALL_CMD_OPTS = ['-y', '-q'] +PIP_INSTALL_CMD_OPTS = ['-q'] -def install(pips): - if not pips: - return +def _cmd_name(distro): + #of course u know they have to change the name... + if distro == settings.RHEL6: + return 'pip-python' + else: + return 'pip' + + +def install(pips, distro): actions = list() pipnames = sorted(pips.keys()) for name in pipnames: @@ -40,20 +48,20 @@ def install(pips): actions.append(pipfull) if actions: LOG.info("Installing python packages [%s]" % (", ".join(actions))) - cmd = INSTALL_CMD + actions + root_cmd = _cmd_name(distro) + cmd = [root_cmd, 'install'] + PIP_INSTALL_CMD_OPTS + actions sh.execute(*cmd, run_as_root=True) -def uninstall(pips): - if not pips: - return +def uninstall(pips, distro): pipnames = sorted(pips.keys()) LOG.info("Uninstalling python packages [%s]" % (", ".join(pipnames))) for name in pipnames: pipinfo = pips.get(name, dict()) - skip_errors = pipinfo.get('skip_uninstall_errors', False) + skip_errors = pipinfo.get('skip_uninstall_errors', True) try: - cmd = UNINSTALL_CMD + [name] + root_cmd = _cmd_name(distro) + cmd = [root_cmd, 'uninstall'] + PIP_UNINSTALL_CMD_OPTS + [name] sh.execute(*cmd, run_as_root=True) except excp.ProcessExecutionError: if skip_errors: diff --git a/devstack/progs/actions.py b/devstack/progs/actions.py index 43d8d494..47f5a9c6 100644 --- a/devstack/progs/actions.py +++ b/devstack/progs/actions.py @@ -48,6 +48,9 @@ _WELCOME_MAP = { # For actions in this list we will reverse the component order _REVERSE_ACTIONS = [settings.UNINSTALL, settings.STOP] +# These will not automatically stop when uninstalled since it seems to break there password reset. +_NO_AUTO_STOP = [settings.DB, settings.RABBIT] + def _clean_action(action): if action is None: @@ -237,16 +240,17 @@ def _run_components(action_name, component_order, components, distro, root_dir, else: results.append(str(start_result)) elif action_name == settings.UNINSTALL: - # always stop first. doesn't hurt if already stopped - but makes - # sure that there are no lingering processes if not - stop_cls = common.get_action_cls(settings.STOP, component) - stop_instance = stop_cls(instances=stop_instances, - distro=distro, - packager=pkg_manager, - config=config, - root=root_dir, - opts=components.get(component, list())) - _stop(component, stop_instance, program_args.get('force', False)) + if component not in _NO_AUTO_STOP: + # always stop first. doesn't hurt if already stopped - but makes + # sure that there are no lingering processes if not + stop_cls = common.get_action_cls(settings.STOP, component) + stop_instance = stop_cls(instances=stop_instances, + distro=distro, + packager=pkg_manager, + config=config, + root=root_dir, + opts=components.get(component, list())) + _stop(component, stop_instance, program_args.get('force', False)) _uninstall(component, instance, program_args.get('force', False)) #display any configs touched... _print_cfgs(config, action_name) diff --git a/devstack/settings.py b/devstack/settings.py index ce862a93..84ee6b41 100644 --- a/devstack/settings.py +++ b/devstack/settings.py @@ -14,14 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. -import operator -import os.path +import os import re -import sys - -from devstack import log as logging - -LOG = logging.getLogger("devstack.settings") # These also have meaning outside python, # ie in the pkg/pip listings so update there also! diff --git a/devstack/shell.py b/devstack/shell.py index 603870db..728d1300 100644 --- a/devstack/shell.py +++ b/devstack/shell.py @@ -38,6 +38,7 @@ def execute(*cmd, **kwargs): check_exit_code = kwargs.pop('check_exit_code', [0]) cwd = kwargs.pop('cwd', None) env_overrides = kwargs.pop('env_overrides', None) + close_stdin = kwargs.pop('close_stdin', False) ignore_exit_code = False if isinstance(check_exit_code, bool): @@ -48,18 +49,22 @@ def execute(*cmd, **kwargs): run_as_root = kwargs.pop('run_as_root', False) shell = kwargs.pop('shell', False) - if run_as_root: - cmd = ROOT_HELPER + list(cmd) - cmd = [str(c) for c in cmd] - execute_cmd = None - str_cmd = " ".join(cmd) + execute_cmd = list() + if run_as_root: + execute_cmd.extend(ROOT_HELPER) + + for c in cmd: + execute_cmd.append(str(c)) + + str_cmd = " ".join(execute_cmd) if shell: execute_cmd = str_cmd.strip() - LOG.debug('Running shell cmd: [%s]' % (str_cmd)) + + if not shell: + LOG.debug('Running cmd: %s' % (execute_cmd)) else: - execute_cmd = cmd - LOG.debug('Running cmd: [%s]' % (str_cmd)) + LOG.debug('Running shell cmd: %s' % (execute_cmd)) if process_input is not None: LOG.debug('With stdin: %s' % (process_input)) @@ -85,9 +90,9 @@ def execute(*cmd, **kwargs): stderr_fh = kwargs.get('stderr_fh') LOG.debug("Redirecting stderr to file handle: %s" % (stderr_fh)) - process_env = env.get() - LOG.debug("With environment: %s" % (process_env)) + process_env = None if env_overrides and len(env_overrides): + process_env = env.get() LOG.debug("With additional environment overrides: %s" % (env_overrides)) for (k, v) in env_overrides.items(): process_env[k] = str(v) @@ -107,7 +112,8 @@ def execute(*cmd, **kwargs): else: result = obj.communicate() - if 'stdin_fh' not in kwargs.keys(): + if (stdin_fh != subprocess.PIPE + and obj.stdin and close_stdin): obj.stdin.close() rc = obj.returncode @@ -155,11 +161,12 @@ def _gen_password(pw_len): return stdout.strip() -def _prompt_password(prompt_=None): - if prompt_: - return getpass.getpass(prompt_) +def prompt_password(pw_prompt=None): + if pw_prompt: + rc = getpass.getpass(pw_prompt) else: - return getpass.getpass() + rc = getpass.getpass() + return rc.strip() def chown_r(path, uid, gid): @@ -181,7 +188,7 @@ def password(prompt_=None, pw_len=8): rd = "" ask_for_pw = env.get_bool(PASS_ASK_ENV, True) if ask_for_pw: - rd = _prompt_password(prompt_) + rd = prompt_password(prompt_) if not rd: return _gen_password(pw_len) else: diff --git a/devstack/utils.py b/devstack/utils.py index 7205b4f0..cb79d6e6 100644 --- a/devstack/utils.py +++ b/devstack/utils.py @@ -34,6 +34,7 @@ from devstack import version PARAM_SUB_REGEX = re.compile(r"%([\w\d]+?)%") EXT_COMPONENT = re.compile(r"^\s*([\w-]+)(?:\((.*)\))?\s*$") +MONTY_PYTHON_TEXT_RE = re.compile("([a-z0-9A-Z!.,'\"]+)") LOG = logging.getLogger("devstack.util") TEMPLATE_EXT = ".tpl" @@ -358,14 +359,12 @@ def color_text(text, color, bold=False): def _color_blob(text, text_color): - special_chars = ['!', '.', ',', "'"] - colored_msg = "" - for ch in text: - if ch.isalpha() or ch.isdigit() or ch in special_chars: - colored_msg += color_text(ch, text_color) - else: - colored_msg += ch - return colored_msg + + def replacer(match): + contents = match.group(1) + return color_text(contents, text_color) + + return MONTY_PYTHON_TEXT_RE.sub(replacer, text) def _goodbye_header(worked):