From 6e4382cb42df0733842f8f274e5d59bdf6e4ca26 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 12 Mar 2012 20:46:21 -0700 Subject: [PATCH] Made it so that the config uses a shared cache (which is shared with the password gen). Adjusted some naming to have slightly shorter variable names (pw_gen). Removed need for section in password generator. Adjusted passing of the renamed variable around and the other places pw_gen should be used. Added helper class that can be used by both pw_gen and cfg to get ids and adjusted print out of the config to now just print out the shared cache, which seems to make sense to me. --- devstack/cfg.py | 59 ++-------------------------- devstack/cfg_helpers.py | 69 +++++++++++++++++++++++++++++++++ devstack/component.py | 25 ++++++------ devstack/components/db.py | 30 +++++--------- devstack/components/glance.py | 11 +++--- devstack/components/keystone.py | 25 ++++++------ devstack/components/melange.py | 7 ++-- devstack/components/nova.py | 11 ++++-- devstack/components/quantum.py | 7 ++-- devstack/components/rabbit.py | 4 +- devstack/env_rc.py | 6 +-- devstack/image/creator.py | 6 +-- devstack/opts.py | 3 +- devstack/passwords.py | 45 +++++++++++++-------- devstack/progs/actions.py | 13 ++++--- devstack/progs/common.py | 19 ++++----- devstack/utils.py | 6 --- stack | 62 +++++++++++++++++++---------- 18 files changed, 226 insertions(+), 182 deletions(-) create mode 100644 devstack/cfg_helpers.py diff --git a/devstack/cfg.py b/devstack/cfg.py index e881a3e5..1f2ac03d 100644 --- a/devstack/cfg.py +++ b/devstack/cfg.py @@ -17,6 +17,7 @@ import re import ConfigParser +from devstack import cfg_helpers from devstack import date from devstack import env from devstack import exceptions as excp @@ -24,7 +25,6 @@ from devstack import log as logging from devstack import shell as sh from devstack import utils - LOG = logging.getLogger("devstack.cfg") ENV_PAT = re.compile(r"^\s*\$\{([\w\d]+):\-(.*)\}\s*$") SUB_MATCH = re.compile(r"(?:\$\(([\w\d]+):([\w\d]+))\)") @@ -68,20 +68,10 @@ class IgnoreMissingConfigParser(ConfigParser.RawConfigParser): return ConfigParser.RawConfigParser.getint(self, section, option) -def make_id(section, option): - joinwhat = [] - if section is not None: - joinwhat.append(str(section)) - if option is not None: - joinwhat.append(str(option)) - return "/".join(joinwhat) - - class StackConfigParser(IgnoreMissingConfigParser): - def __init__(self): + def __init__(self, cache): IgnoreMissingConfigParser.__init__(self) - self.configs_fetched = dict() - self.db_dsns = dict() + self.configs_fetched = cache def _resolve_value(self, section, option, value_gotten): if section == 'host' and option == 'ip': @@ -98,7 +88,7 @@ class StackConfigParser(IgnoreMissingConfigParser): return val def get(self, section, option): - key = make_id(section, option) + key = cfg_helpers.make_id(section, option) if key in self.configs_fetched: value = self.configs_fetched.get(key) LOG.debug("Fetched cached value [%s] for param [%s]" % (value, key)) @@ -110,11 +100,6 @@ class StackConfigParser(IgnoreMissingConfigParser): self.configs_fetched[key] = value return value - def set(self, section, option, value): - key = make_id(section, option) - self.configs_fetched[key] = value - return IgnoreMissingConfigParser.set(self, section, option, value) - def _resolve_replacements(self, value): LOG.debug("Performing simple replacement on [%s]", value) @@ -151,42 +136,6 @@ class StackConfigParser(IgnoreMissingConfigParser): LOG.debug("Using raw config provided value [%s]" % (extracted_val)) return extracted_val - def get_dbdsn(self, dbname): - #check the dsn cache - if dbname in self.db_dsns: - return self.db_dsns[dbname] - user = self.get("db", "sql_user") - host = self.get("db", "sql_host") - port = self.get("db", "port") - pw = self.get("passwords", "sql") - #form the dsn (from components we have...) - #dsn = "://:@:/" - if not host: - msg = "Unable to fetch a database dsn - no sql host found" - raise excp.BadParamException(msg) - driver = self.get("db", "type") - if not driver: - msg = "Unable to fetch a database dsn - no db driver type found" - raise excp.BadParamException(msg) - dsn = driver + "://" - if user: - dsn += user - if pw: - dsn += ":" + pw - if user or pw: - dsn += "@" - dsn += host - if port: - dsn += ":" + port - if dbname: - dsn += "/" + dbname - else: - dsn += "/" - LOG.debug("For database [%s] fetched dsn [%s] %s" % (dbname, dsn, CACHE_MSG)) - #store for later... - self.db_dsns[dbname] = dsn - return dsn - def add_header(fn, contents): lines = list() diff --git a/devstack/cfg_helpers.py b/devstack/cfg_helpers.py new file mode 100644 index 00000000..cd64cb61 --- /dev/null +++ b/devstack/cfg_helpers.py @@ -0,0 +1,69 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from devstack import exceptions as excp +from devstack import log as logging +from devstack import settings + +LOG = logging.getLogger("devstack.cfg.helpers") + + +def make_id(section, option): + joinwhat = [] + if section is not None: + joinwhat.append(str(section)) + if option is not None: + joinwhat.append(str(option)) + return "/".join(joinwhat) + + +def fetch_run_type(config): + run_type = config.getdefaulted("default", "run_type", settings.RUN_TYPE_DEF) + run_type = run_type.upper() + return run_type + + +def fetch_dbdsn(config, pw_gen, dbname=''): + #check the dsn cache + user = config.get("db", "sql_user") + host = config.get("db", "sql_host") + port = config.get("db", "port") + pw = pw_gen.get_password("sql") + #form the dsn (from components we have...) + #dsn = "://:@:/" + if not host: + msg = "Unable to fetch a database dsn - no sql host found" + raise excp.BadParamException(msg) + driver = config.get("db", "type") + if not driver: + msg = "Unable to fetch a database dsn - no db driver type found" + raise excp.BadParamException(msg) + dsn = driver + "://" + if user: + dsn += user + if pw: + dsn += ":" + pw + if user or pw: + dsn += "@" + dsn += host + if port: + dsn += ":" + port + if dbname: + dsn += "/" + dbname + else: + dsn += "/" + LOG.debug("For database [%s] fetched dsn [%s]" % (dbname, dsn)) + return dsn diff --git a/devstack/component.py b/devstack/component.py index e16027e6..ad3d5bca 100644 --- a/devstack/component.py +++ b/devstack/component.py @@ -16,7 +16,7 @@ import weakref -from devstack import cfg +from devstack import cfg_helpers from devstack import downloader as down from devstack import exceptions as excp from devstack import log as logging @@ -53,19 +53,22 @@ BASE_LINK_DIR = "/etc" class ComponentBase(object): - def __init__(self, component_name, runner, root, opts, instances=None, + def __init__(self, component_name, runner, + root_dir, component_options, + instances=None, **kwds): self.component_name = component_name + # The runner has a reference to us, so use a weakref here to # avoid breaking garbage collection. self.runner = weakref.proxy(runner) - self.root = root - self.component_opts = opts or [] + self.root = root_dir + self.component_opts = component_options or [] self.instances = instances or {} # Parts of the global runner context that we use self.cfg = runner.cfg - self.password_generator = runner.password_generator + self.pw_gen = runner.pw_gen self.packager = runner.pkg_manager self.distro = runner.distro @@ -79,9 +82,7 @@ class ComponentBase(object): self.kargs = kwds def get_dependencies(self): - deps = settings.COMPONENT_DEPENDENCIES.get(self.component_name) - if not deps: - return list() + deps = settings.COMPONENT_DEPENDENCIES.get(self.component_name) or list() return list(deps) def verify(self): @@ -124,13 +125,13 @@ class PkgInstallComponent(ComponentBase): branch = self.cfg.get(cfg_section, cfg_key) if not branch: msg = "No branch entry found at config location [%s]" % \ - (cfg.make_id(cfg_section, cfg_key)) + (cfg_helpers.make_id(cfg_section, cfg_key)) raise excp.ConfigException(msg) (cfg_section, cfg_key) = uri_tuple uri = self.cfg.get(cfg_section, cfg_key) if not uri: msg = "No uri entry found at config location [%s]" % \ - (cfg.make_id(cfg_section, cfg_key)) + (cfg_helpers.make_id(cfg_section, cfg_key)) raise excp.ConfigException(msg) self.tracewriter.download_happened(target_loc, uri) dirs_made = down.download(target_loc, uri, branch) @@ -450,7 +451,7 @@ class ProgramRuntime(ComponentBase): def configure(self): # First make a pass and make sure all runtime (e.g. upstart) # config files are in place.... - cls = RUNNER_CLS_MAPPING[utils.fetch_run_type(self.cfg)] + cls = RUNNER_CLS_MAPPING[cfg_helpers.fetch_run_type(self.cfg)] instance = cls(self.cfg, self.component_name, self.tracedir) tot_am = 0 for app_info in self._get_apps_to_start(): @@ -473,7 +474,7 @@ class ProgramRuntime(ComponentBase): def start(self): # Select how we are going to start it - cls = RUNNER_CLS_MAPPING[utils.fetch_run_type(self.cfg)] + cls = RUNNER_CLS_MAPPING[cfg_helpers.fetch_run_type(self.cfg)] instance = cls(self.cfg, self.component_name, self.tracedir) am_started = 0 for app_info in self._get_apps_to_start(): diff --git a/devstack/components/db.py b/devstack/components/db.py index 5e9c2de8..01febcd3 100644 --- a/devstack/components/db.py +++ b/devstack/components/db.py @@ -85,9 +85,6 @@ REQ_PKGS = ['db.json'] #config keys we warm up so u won't be prompted later WARMUP_PWS = ['sql'] -#partial of database user prompt -PASSWORD_DESCRIPTION = 'the database user' - class DBUninstaller(comp.PkgUninstallComponent): def __init__(self, *args, **kargs): @@ -96,8 +93,7 @@ class DBUninstaller(comp.PkgUninstallComponent): def warm_configs(self): for pw_key in WARMUP_PWS: - self.password_generator.get_password( - 'passwords', pw_key, PASSWORD_DESCRIPTION) + self.pw_gen.get_password(pw_key) def pre_uninstall(self): dbtype = self.cfg.get("db", "type") @@ -110,10 +106,8 @@ class DBUninstaller(comp.PkgUninstallComponent): if pwd_cmd: LOG.info("Ensuring your database is started before we operate on it.") self.runtime.restart() - old_pw = self.password_generator.get_password( - 'passwords', 'sql', PASSWORD_DESCRIPTION) params = { - 'OLD_PASSWORD': old_pw, + 'OLD_PASSWORD': self.pw_gen.get_password('sql'), 'NEW_PASSWORD': RESET_BASE_PW, 'USER': self.cfg.getdefaulted("db", "sql_user", 'root'), } @@ -135,8 +129,7 @@ class DBInstaller(comp.PkgInstallComponent): #in pre-install and post-install sections host_ip = self.cfg.get('host', 'ip') out = { - 'PASSWORD': self.password_generator.get_password( - "passwords", "sql", PASSWORD_DESCRIPTION), + 'PASSWORD': self.pw_gen.get_password("sql"), 'BOOT_START': ("%s" % (True)).lower(), 'USER': self.cfg.getdefaulted("db", "sql_user", 'root'), 'SERVICE_HOST': host_ip, @@ -146,8 +139,7 @@ class DBInstaller(comp.PkgInstallComponent): def warm_configs(self): for pw_key in WARMUP_PWS: - self.password_generator.get_password( - 'passwords', pw_key, PASSWORD_DESCRIPTION) + self.pw_gen.get_password(pw_key) def _configure_db_confs(self): dbtype = self.cfg.get("db", "type") @@ -200,8 +192,7 @@ class DBInstaller(comp.PkgInstallComponent): LOG.info("Ensuring your database is started before we operate on it.") self.runtime.restart() params = { - 'NEW_PASSWORD': self.password_generator.get_password( - "passwords", "sql", PASSWORD_DESCRIPTION), + 'NEW_PASSWORD': self.pw_gen.get_password("sql"), 'USER': self.cfg.getdefaulted("db", "sql_user", 'root'), 'OLD_PASSWORD': RESET_BASE_PW, } @@ -220,8 +211,7 @@ class DBInstaller(comp.PkgInstallComponent): LOG.info("Ensuring your database is started before we operate on it.") self.runtime.restart() params = { - 'PASSWORD': self.password_generator.get_password( - "passwords", "sql", PASSWORD_DESCRIPTION), + 'PASSWORD': self.pw_gen.get_password("sql"), 'USER': user, } cmds = [{'cmd': grant_cmd}] @@ -296,13 +286,13 @@ class DBRuntime(comp.EmptyRuntime): return comp.STATUS_UNKNOWN -def drop_db(cfg, dbname): +def drop_db(cfg, pw_gen, dbname): dbtype = cfg.get("db", "type") dbactions = DB_ACTIONS.get(dbtype) if dbactions and dbactions.get('drop_db'): dropcmd = dbactions.get('drop_db') params = dict() - params['PASSWORD'] = cfg.get("passwords", "sql") + params['PASSWORD'] = pw_gen.get_password("sql") params['USER'] = cfg.getdefaulted("db", "sql_user", 'root') params['DB'] = dbname cmds = list() @@ -316,13 +306,13 @@ def drop_db(cfg, dbname): raise NotImplementedError(msg) -def create_db(cfg, dbname): +def create_db(cfg, pw_gen, dbname): dbtype = cfg.get("db", "type") dbactions = DB_ACTIONS.get(dbtype) if dbactions and dbactions.get('create_db'): createcmd = dbactions.get('create_db') params = dict() - params['PASSWORD'] = cfg.get("passwords", "sql") + params['PASSWORD'] = pw_gen.get_password("sql") params['USER'] = cfg.getdefaulted("db", "sql_user", 'root') params['DB'] = dbname cmds = list() diff --git a/devstack/components/glance.py b/devstack/components/glance.py index a33b2dee..1722ecb8 100644 --- a/devstack/components/glance.py +++ b/devstack/components/glance.py @@ -17,6 +17,7 @@ import io from devstack import cfg +from devstack import cfg_helpers from devstack import component as comp from devstack import log as logging from devstack import settings @@ -118,8 +119,8 @@ class GlanceInstaller(comp.PythonInstallComponent): def _setup_db(self): LOG.info("Fixing up database named %s.", DB_NAME) - db.drop_db(self.cfg, DB_NAME) - db.create_db(self.cfg, DB_NAME) + db.drop_db(self.cfg, self.pw_gen, DB_NAME) + db.create_db(self.cfg, self.pw_gen, DB_NAME) def _get_source_config(self, config_fn): if config_fn == POLICY_JSON: @@ -184,10 +185,10 @@ class GlanceInstaller(comp.PythonInstallComponent): mp = dict() mp['DEST'] = self.appdir mp['SYSLOG'] = self.cfg.getboolean("default", "syslog") - mp['SQL_CONN'] = self.cfg.get_dbdsn(DB_NAME) + mp['SQL_CONN'] = cfg_helpers.fetch_dbdsn(self.cfg, self.pw_gen, DB_NAME) mp['SERVICE_HOST'] = self.cfg.get('host', 'ip') mp['HOST_IP'] = self.cfg.get('host', 'ip') - mp.update(keystone.get_shared_params(self.cfg, self.password_generator, 'glance')) + mp.update(keystone.get_shared_params(self.cfg, self.pw_gen, 'glance')) return mp @@ -224,4 +225,4 @@ class GlanceRuntime(comp.PythonRuntime): # TODO: make this less cheesy - need to wait till glance goes online LOG.info("Waiting %s seconds so that glance can start up before image install." % (WAIT_ONLINE_TO)) sh.sleep(WAIT_ONLINE_TO) - creator.ImageCreationService(self.cfg, self.password_generator).install() + creator.ImageCreationService(self.cfg, self.pw_gen).install() diff --git a/devstack/components/keystone.py b/devstack/components/keystone.py index f7f7353e..6042ad98 100644 --- a/devstack/components/keystone.py +++ b/devstack/components/keystone.py @@ -19,6 +19,7 @@ import io from urlparse import urlunparse from devstack import cfg +from devstack import cfg_helpers from devstack import component as comp from devstack import log as logging from devstack import settings @@ -132,8 +133,8 @@ class KeystoneInstaller(comp.PythonInstallComponent): def _setup_db(self): LOG.info("Fixing up database named %s.", DB_NAME) - db.drop_db(self.cfg, DB_NAME) - db.create_db(self.cfg, DB_NAME) + db.drop_db(self.cfg, self.pw_gen, DB_NAME) + db.create_db(self.cfg, self.pw_gen, DB_NAME) def _setup_initer(self): LOG.info("Configuring keystone initializer template %s.", MANAGE_DATA_CONF) @@ -191,7 +192,7 @@ class KeystoneInstaller(comp.PythonInstallComponent): return comp.PythonInstallComponent._get_source_config(self, config_fn) def warm_configs(self): - get_shared_params(self.cfg, self.password_generator) + get_shared_params(self.cfg, self.pw_gen) def _get_param_map(self, config_fn): #these be used to fill in the configuration/cmds + @@ -202,11 +203,11 @@ class KeystoneInstaller(comp.PythonInstallComponent): mp['BIN_DIR'] = self.bindir mp['CONFIG_FILE'] = sh.joinpths(self.cfgdir, ROOT_CONF) if config_fn == ROOT_CONF: - mp['SQL_CONN'] = self.cfg.get_dbdsn(DB_NAME) + mp['SQL_CONN'] = cfg_helpers.fetch_dbdsn(self.cfg, self.pw_gen, DB_NAME) mp['KEYSTONE_DIR'] = self.appdir - mp.update(get_shared_params(self.cfg, self.password_generator)) + mp.update(get_shared_params(self.cfg, self.pw_gen)) elif config_fn == MANAGE_DATA_CONF: - mp.update(get_shared_params(self.cfg, self.password_generator)) + mp.update(get_shared_params(self.cfg, self.pw_gen)) return mp @@ -248,8 +249,7 @@ class KeystoneRuntime(comp.PythonRuntime): return APP_OPTIONS.get(app) -def get_shared_params(config, password_generator, service_user_name=None): - LOG.debug('password_generator %s', password_generator) +def get_shared_params(config, pw_gen, service_user_name=None): mp = dict() host_ip = config.get('host', 'ip') @@ -263,18 +263,15 @@ def get_shared_params(config, password_generator, service_user_name=None): mp['DEMO_TENANT_NAME'] = mp['DEMO_USER_NAME'] #tokens and passwords - mp['SERVICE_TOKEN'] = password_generator.get_password( - 'passwords', + mp['SERVICE_TOKEN'] = pw_gen.get_password( "service_token", 'the service admin token', ) - mp['ADMIN_PASSWORD'] = password_generator.get_password( - 'passwords', + mp['ADMIN_PASSWORD'] = pw_gen.get_password( 'horizon_keystone_admin', 'the horizon and keystone admin', 20) - mp['SERVICE_PASSWORD'] = password_generator.get_password( - 'passwords', + mp['SERVICE_PASSWORD'] = pw_gen.get_password( 'service_password', 'service authentication', ) diff --git a/devstack/components/melange.py b/devstack/components/melange.py index fb62dca9..415e3a01 100644 --- a/devstack/components/melange.py +++ b/devstack/components/melange.py @@ -17,6 +17,7 @@ import io from devstack import cfg +from devstack import cfg_helpers from devstack import component as comp from devstack import log as logging from devstack import settings @@ -88,8 +89,8 @@ class MelangeInstaller(comp.PythonInstallComponent): def _setup_db(self): LOG.info("Fixing up database named %s.", DB_NAME) - db.drop_db(self.cfg, DB_NAME) - db.create_db(self.cfg, DB_NAME) + db.drop_db(self.cfg, self.pw_gen, DB_NAME) + db.create_db(self.cfg, self.pw_gen, DB_NAME) def _get_pkgs(self): return list(REQ_PKGS) @@ -115,7 +116,7 @@ class MelangeInstaller(comp.PythonInstallComponent): with io.BytesIO(contents) as stream: config = cfg.IgnoreMissingConfigParser() config.readfp(stream) - db_dsn = self.cfg.get_dbdsn(DB_NAME) + db_dsn = cfg_helpers.fetch_dbdsn(self.cfg, self.pw_gen, DB_NAME) old_dbsn = config.get('DEFAULT', 'sql_connection') if db_dsn != old_dbsn: config.set('DEFAULT', 'sql_connection', db_dsn) diff --git a/devstack/components/nova.py b/devstack/components/nova.py index 23ba0824..0758879c 100644 --- a/devstack/components/nova.py +++ b/devstack/components/nova.py @@ -16,6 +16,7 @@ from urlparse import urlunparse +from devstack import cfg_helpers from devstack import component as comp from devstack import date from devstack import exceptions @@ -371,8 +372,8 @@ class NovaInstaller(comp.PythonInstallComponent): def _setup_db(self): LOG.info("Fixing up database named %s.", DB_NAME) - db.drop_db(self.cfg, DB_NAME) - db.create_db(self.cfg, DB_NAME) + db.drop_db(self.cfg, self.pw_gen, DB_NAME) + db.create_db(self.cfg, self.pw_gen, DB_NAME) def _generate_nova_conf(self): LOG.info("Generating dynamic content for nova configuration (%s)." % (API_CONF)) @@ -406,7 +407,7 @@ class NovaInstaller(comp.PythonInstallComponent): mp['FIXED_NETWORK_SIZE'] = self.cfg.getdefaulted('nova', 'fixed_network_size', '256') mp['FIXED_RANGE'] = self.cfg.getdefaulted('nova', 'fixed_range', '10.0.0.0/24') else: - mp.update(keystone.get_shared_params(self.cfg, self.password_generator, 'nova')) + mp.update(keystone.get_shared_params(self.cfg, self.pw_gen, 'nova')) return mp def configure(self): @@ -574,6 +575,7 @@ class NovaVolumeConfigurator(object): class NovaConfConfigurator(object): def __init__(self, ni): self.cfg = ni.cfg + self.pw_gen = ni.pw_gen self.instances = ni.instances self.component_root = ni.component_root self.appdir = ni.appdir @@ -637,7 +639,8 @@ class NovaConfConfigurator(object): nova_conf.add('my_ip', hostip) #setup your sql connection - nova_conf.add('sql_connection', self.cfg.get_dbdsn('nova')) + db_dsn = cfg_helpers.fetch_dbdsn(self.cfg, self.pw_gen, DB_NAME) + nova_conf.add('sql_connection', db_dsn) #configure anything libvirt releated? virt_driver = _canon_virt_driver(self._getstr('virt_driver')) diff --git a/devstack/components/quantum.py b/devstack/components/quantum.py index b7ac1b03..0a2ed280 100644 --- a/devstack/components/quantum.py +++ b/devstack/components/quantum.py @@ -17,6 +17,7 @@ import io from devstack import cfg +from devstack import cfg_helpers from devstack import component as comp from devstack import log as logging from devstack import settings @@ -162,7 +163,7 @@ class QuantumInstaller(comp.PkgInstallComponent): config.readfp(stream) db_dsn = config.get("DATABASE", "sql_connection") if db_dsn: - generated_dsn = self.cfg.get_dbdsn(DB_NAME) + generated_dsn = cfg_helpers.fetch_dbdsn(self.cfg, self.pw_gen, DB_NAME) if generated_dsn != db_dsn: config.set("DATABASE", "sql_connection", generated_dsn) with io.BytesIO() as outputstream: @@ -199,8 +200,8 @@ class QuantumInstaller(comp.PkgInstallComponent): def _setup_db(self): LOG.info("Fixing up database named %s.", DB_NAME) - db.drop_db(self.cfg, DB_NAME) - db.create_db(self.cfg, DB_NAME) + db.drop_db(self.cfg, self.pw_gen, DB_NAME) + db.create_db(self.cfg, self.pw_gen, DB_NAME) def _get_source_config(self, config_fn): if config_fn == PLUGIN_CONF: diff --git a/devstack/components/rabbit.py b/devstack/components/rabbit.py index 6eac0d79..c5335ea2 100644 --- a/devstack/components/rabbit.py +++ b/devstack/components/rabbit.py @@ -71,12 +71,12 @@ class RabbitInstaller(comp.PkgInstallComponent): def warm_configs(self): for pw_key in WARMUP_PWS: - self.password_generator.get_password("passwords", pw_key, PW_USER_PROMPT) + self.pw_gen.get_password(pw_key, PW_USER_PROMPT) def _setup_pw(self): LOG.info("Setting up your rabbit-mq guest password.") self.runtime.restart() - passwd = self.password_generator.get_password('passwords', "rabbit", PW_USER_PROMPT) + passwd = self.pw_gen.get_password("rabbit", PW_USER_PROMPT) cmd = PWD_CMD + [passwd] sh.execute(*cmd, run_as_root=True) LOG.info("Restarting so that your rabbit-mq guest password is reflected.") diff --git a/devstack/env_rc.py b/devstack/env_rc.py index 22adb266..a0094632 100644 --- a/devstack/env_rc.py +++ b/devstack/env_rc.py @@ -51,9 +51,9 @@ QUOTED_PAT = re.compile(r"^\s*[\"](.*)[\"]\s*$") class RcWriter(object): - def __init__(self, cfg, password_generator): + def __init__(self, cfg, pw_gen): self.cfg = cfg - self.password_generator = password_generator + self.pw_gen = pw_gen def _make_export(self, export_name, value): escaped_val = sh.shellquote(value) @@ -153,7 +153,7 @@ class RcWriter(object): sh.write_file(fn, contents) def _get_os_envs(self): - key_params = keystone.get_shared_params(self.cfg, self.password_generator) + key_params = keystone.get_shared_params(self.cfg, self.pw_gen) to_set = dict() to_set['OS_PASSWORD'] = key_params['ADMIN_PASSWORD'] to_set['OS_TENANT_NAME'] = key_params['DEMO_TENANT_NAME'] diff --git a/devstack/image/creator.py b/devstack/image/creator.py index 25411368..f48bad58 100644 --- a/devstack/image/creator.py +++ b/devstack/image/creator.py @@ -211,14 +211,14 @@ class ImageRegistry: class ImageCreationService: - def __init__(self, cfg, password_generator): + def __init__(self, cfg, pw_gen): self.cfg = cfg - self.password_generator = password_generator + self.pw_gen = pw_gen def _get_token(self): LOG.info("Fetching your keystone admin token so that we can perform image uploads.") - key_params = keystone.get_shared_params(self.cfg, self.password_generator) + key_params = keystone.get_shared_params(self.cfg, self.pw_gen) keystone_service_url = key_params['SERVICE_ENDPOINT'] keystone_token_url = "%s/tokens" % (keystone_service_url) diff --git a/devstack/opts.py b/devstack/opts.py index 9f165730..f9afc7a6 100644 --- a/devstack/opts.py +++ b/devstack/opts.py @@ -53,7 +53,8 @@ def parse(): action="store_true", dest="dryrun", default=False, - help="perform actions without actually doing any of them and/or attempt to do this: (default: %default)") + help=("perform ACTION but do not actually run any of the commands" + " that would normally complete ACTION: (default: %default)")) base_group = OptionGroup(parser, "Install & uninstall & start & stop specific options") base_group.add_option("-a", "--action", diff --git a/devstack/passwords.py b/devstack/passwords.py index 21b9774a..60bec5f6 100644 --- a/devstack/passwords.py +++ b/devstack/passwords.py @@ -7,9 +7,17 @@ import logging import os import re -from devstack.cfg import make_id +from devstack import cfg_helpers LOG = logging.getLogger("devstack.passwords") +PW_SECTION = 'passwords' +HELPFUL_DESCRIPTIONS = { + 'sql': 'the database user', +} + + +def get_pw_usage(option): + return HELPFUL_DESCRIPTIONS.get(option, '???') def generate_random(length): @@ -21,18 +29,16 @@ def generate_random(length): class PasswordGenerator(object): - def __init__(self, cfg, prompt_user=True): + def __init__(self, kv_cache, cfg, + prompt_user=True): self.cfg = cfg + self.config_cache = kv_cache self.prompt_user = prompt_user - # Store the values accessed by the caller - # so the main script can print them out - # at the end. - self.accessed = {} def _prompt_user(self, prompt_text): LOG.debug('Asking the user for a %r password', prompt_text) message = ("Enter a password to use for %s " - "[or press enter to get a generated one] " % prompt_text + "[or press enter to get a generated one]: " % prompt_text ) rc = "" while True: @@ -50,21 +56,27 @@ class PasswordGenerator(object): break return rc - # FIXME: Remove the "section" argument, since it is always the same. - def get_password(self, section, option, prompt_text, length=8): + def get_password(self, option, prompt_text=None, length=8): """Returns a password identified by the configuration location.""" + + if not prompt_text: + prompt_text = get_pw_usage(option) + LOG.debug('Looking for password %s (%s)', option, prompt_text) + cache_key = cfg_helpers.make_id(PW_SECTION, option) + password = self.config_cache.get(cache_key) + # Look in the configuration file(s) - try: - password = self.cfg.get(section, option) - except ConfigParser.Error: - password = '' + if not password: + try: + password = self.cfg.get(PW_SECTION, option) + except ConfigParser.Error: + password = '' # Optionally ask the user if not password and self.prompt_user: password = self._prompt_user(prompt_text) - self.accessed[make_id(section, option)] = password # If we still don't have a value, make one up. if not password: @@ -72,7 +84,8 @@ class PasswordGenerator(object): option, prompt_text) password = generate_random(length) - # Update the configration cache so that other parts of the + # Update the cache so that other parts of the # code can find the value. - self.cfg.set(section, option, password) + self.config_cache[cache_key] = password + return password diff --git a/devstack/progs/actions.py b/devstack/progs/actions.py index b29eee96..ff487b35 100644 --- a/devstack/progs/actions.py +++ b/devstack/progs/actions.py @@ -139,13 +139,13 @@ PREQ_ACTIONS = { class ActionRunner(object): def __init__(self, distro, action, directory, config, - password_generator, pkg_manager, + pw_gen, pkg_manager, **kargs): self.distro = distro self.action = action self.directory = directory self.cfg = config - self.password_generator = password_generator + self.pw_gen = pw_gen self.pkg_manager = pkg_manager self.kargs = kargs self.components = dict() @@ -205,8 +205,8 @@ class ActionRunner(object): # the component keep a weakref to it. instance = cls(instances=all_instances, runner=self, - root=self.directory, - opts=components.get(component, list()), + root_dir=self.directory, + component_options=components.get(component), keep_old=self.kargs.get("keep_old") ) all_instances[component] = instance @@ -225,7 +225,8 @@ class ActionRunner(object): if preq_components: LOG.info("Having to activate prerequisite action [%s] for %s components." % (preq_action, len(preq_components))) preq_runner = ActionRunner(self.distro, preq_action, - self.directory, self.cfg, self.pkg_manager, + self.directory, self.cfg, self.pw_gen, + self.pkg_manager, components=preq_components, **self.kargs) preq_runner.run() @@ -248,7 +249,7 @@ class ActionRunner(object): inst = instances[component] inst.warm_configs() if self.gen_rc and self.rc_file: - writer = env_rc.RcWriter(self.cfg, self.password_generator) + writer = env_rc.RcWriter(self.cfg, self.pw_gen) if not sh.isfile(self.rc_file): LOG.info("Generating a file at [%s] that will contain your environment settings." % (self.rc_file)) writer.write(self.rc_file) diff --git a/devstack/progs/common.py b/devstack/progs/common.py index 20f5afbe..799679b2 100644 --- a/devstack/progs/common.py +++ b/devstack/progs/common.py @@ -155,10 +155,12 @@ def get_packager(distro, keep_packages): return cls(distro, keep_packages) -def get_config(cfg_fn=None): +def get_config(cfg_fn=None, kv_cache=None): if not cfg_fn: cfg_fn = sh.canon_path(settings.STACK_CONFIG_LOCATION) - config_instance = cfg.StackConfigParser() + if kv_cache is None: + kv_cache = dict() + config_instance = cfg.StackConfigParser(kv_cache) config_instance.read(cfg_fn) return config_instance @@ -174,16 +176,15 @@ def get_components_deps(runner, root_dir = root_dir or _FAKE_ROOT_DIR while len(active_names): component = active_names.pop() - component_opts = base_components.get(component) or [] + component_opts = base_components.get(component) or list() cls = get_action_cls(action_name, component, distro) - instance = cls(instances=[], + instance = cls(instances=list(), runner=runner, - root=root_dir, - opts=component_opts, + root_dir=root_dir, + component_options=component_opts, + keep_old=False ) - deps = instance.get_dependencies() - if deps is None: - deps = set() + deps = instance.get_dependencies() or set() all_components[component] = set(deps) for d in deps: if d not in all_components and d not in active_names: diff --git a/devstack/utils.py b/devstack/utils.py index 24453305..7e4aade0 100644 --- a/devstack/utils.py +++ b/devstack/utils.py @@ -206,12 +206,6 @@ def get_host_ip(): return ip -def fetch_run_type(config): - run_type = config.getdefaulted("default", "run_type", settings.RUN_TYPE_DEF) - run_type = run_type.upper() - return run_type - - def is_interface(intfc): if intfc in get_interfaces(): return True diff --git a/stack b/stack index a9ef39ce..1bc0fef0 100755 --- a/stack +++ b/stack @@ -20,6 +20,7 @@ import sys import time import traceback +from devstack import cfg_helpers from devstack import colorlog from devstack import date from devstack import log as logging @@ -32,6 +33,7 @@ from devstack import utils from devstack.progs import actions from devstack.progs import common + LOG = logging.getLogger("devstack.stack") # This is used to map an action to a useful string for @@ -43,8 +45,18 @@ _WELCOME_MAP = { settings.STOP: "STOPPER", } +_CFG_GROUPS = { + cfg_helpers.make_id('passwords', None): 'Passwords', + cfg_helpers.make_id('db', None): 'Database info', + #catch all + cfg_helpers.make_id(None, None): 'Misc configs', +} -def dump_config(config_obj, password_generator): +_CFG_ORDERING = sorted(_CFG_GROUPS.keys()) +_CFG_ORDERING.reverse() + + +def dump_config(config_cache): def item_format(key, value): return "\t%s=%s" % (str(key), str(value)) @@ -54,21 +66,26 @@ def dump_config(config_obj, password_generator): value = mp.get(key) LOG.info(item_format(key, value)) - passwords_gotten = password_generator.accessed - full_cfgs = config_obj.configs_fetched - db_dsns = config_obj.db_dsns - if passwords_gotten or full_cfgs or db_dsns: - if passwords_gotten: - LOG.info("Passwords:") - map_print(passwords_gotten) - if full_cfgs: - filtered = dict((k, v) for (k, v) in full_cfgs.items() if k not in passwords_gotten) - if filtered: - LOG.info("Configs:") - map_print(filtered) - if db_dsns: - LOG.info("Data source names:") - map_print(db_dsns) + #first partition into our groups + partitions = dict() + for name in _CFG_ORDERING: + partitions[name] = dict() + + #now put the config cached values into there partition + for (k, v) in config_cache.items(): + for name in _CFG_ORDERING: + entries = partitions[name] + if k.startswith(name): + entries[k] = v + break + + #now print them.. + for name in _CFG_ORDERING: + nice_name = _CFG_GROUPS.get(name) + LOG.info(nice_name + ":") + entries = partitions.get(name) + if entries: + map_print(entries) def run(args): @@ -91,20 +108,25 @@ def run(args): #here on out we should be using the logger (and not print) start_time = time.time() - config = common.get_config() # Stash the dryrun value (if any) into the global configuration sh.set_dryrun(args['dryrun']) - password_generator = passwords.PasswordGenerator(config, args['prompt_for_passwords']) + + config_kv_cache = dict() + config = common.get_config(kv_cache=config_kv_cache) + pw_gen = passwords.PasswordGenerator(config_kv_cache, config, args['prompt_for_passwords']) pkg_manager = common.get_packager(distro, args['keep_old']) + components = utils.parse_components(args.pop("components")) - runner = actions.ActionRunner(distro, action, rootdir, config, password_generator, + runner = actions.ActionRunner(distro, action, rootdir, config, pw_gen, pkg_manager, components=components, **args) + LOG.info("Starting action [%s] on %s for distro [%s]" % (action, date.rcf8222date(), distro)) runner.run() LOG.info("It took (%s) to complete action [%s]" % (common.format_secs_taken((time.time() - start_time)), action)) LOG.info("After action [%s] your settings which were created or read are:" % (action)) - dump_config(config, password_generator) + + dump_config(config_kv_cache) return True