x/anvil
Code Issues Proposed changes

Made it so that the config uses a shared cache (which is shared with the password gen).

Browse Source 
Adjusted some naming to have slightly shorter variable names (pw_gen).
Removed need for section in password generator.
Adjusted passing of the renamed variable around and the other places pw_gen should be used.

Added helper class that can be used by both pw_gen and cfg to get ids and adjusted print out of the config to
now just print out the shared cache, which seems to make sense to me.
This commit is contained in:
Joshua Harlow 2012-03-12 20:46:21 -07:00
parent 6e41601616
commit 6e4382cb42
18 changed files with 226 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
devstack/cfg.py
View File

@ -17,6 +17,7 @@
import re import re
import ConfigParser import ConfigParser
from devstack import cfg_helpers
from devstack import date from devstack import date
from devstack import env from devstack import env
from devstack import exceptions as excp from devstack import exceptions as excp
@ -24,7 +25,6 @@ from devstack import log as logging
from devstack import shell as sh from devstack import shell as sh
from devstack import utils from devstack import utils
LOG = logging.getLogger("devstack.cfg") LOG = logging.getLogger("devstack.cfg")
ENV_PAT = re.compile(r"^\s*\$\{([\w\d]+):\-(.*)\}\s*$") ENV_PAT = re.compile(r"^\s*\$\{([\w\d]+):\-(.*)\}\s*$")
SUB_MATCH = re.compile(r"(?:\$\(([\w\d]+):([\w\d]+))\)") SUB_MATCH = re.compile(r"(?:\$\(([\w\d]+):([\w\d]+))\)")
@ -68,20 +68,10 @@ class IgnoreMissingConfigParser(ConfigParser.RawConfigParser):
return ConfigParser.RawConfigParser.getint(self, section, option) return ConfigParser.RawConfigParser.getint(self, section, option)
def make_id(section, option):
joinwhat = []
if section is not None:
joinwhat.append(str(section))
if option is not None:
joinwhat.append(str(option))
return "/".join(joinwhat)
class StackConfigParser(IgnoreMissingConfigParser): class StackConfigParser(IgnoreMissingConfigParser):
def __init__(self): def __init__(self, cache):
IgnoreMissingConfigParser.__init__(self) IgnoreMissingConfigParser.__init__(self)
self.configs_fetched = dict() self.configs_fetched = cache
self.db_dsns = dict()
def _resolve_value(self, section, option, value_gotten): def _resolve_value(self, section, option, value_gotten):
if section == 'host' and option == 'ip': if section == 'host' and option == 'ip':
@ -98,7 +88,7 @@ class StackConfigParser(IgnoreMissingConfigParser):
return val return val
def get(self, section, option): def get(self, section, option):
key = make_id(section, option) key = cfg_helpers.make_id(section, option)
if key in self.configs_fetched: if key in self.configs_fetched:
value = self.configs_fetched.get(key) value = self.configs_fetched.get(key)
LOG.debug("Fetched cached value [%s] for param [%s]" % (value, key)) LOG.debug("Fetched cached value [%s] for param [%s]" % (value, key))
@ -110,11 +100,6 @@ class StackConfigParser(IgnoreMissingConfigParser):
self.configs_fetched[key] = value self.configs_fetched[key] = value
return value return value
def set(self, section, option, value):
key = make_id(section, option)
self.configs_fetched[key] = value
return IgnoreMissingConfigParser.set(self, section, option, value)
def _resolve_replacements(self, value): def _resolve_replacements(self, value):
LOG.debug("Performing simple replacement on [%s]", value) LOG.debug("Performing simple replacement on [%s]", value)
@ -151,42 +136,6 @@ class StackConfigParser(IgnoreMissingConfigParser):
LOG.debug("Using raw config provided value [%s]" % (extracted_val)) LOG.debug("Using raw config provided value [%s]" % (extracted_val))
return extracted_val return extracted_val
def get_dbdsn(self, dbname):
#check the dsn cache
if dbname in self.db_dsns:
return self.db_dsns[dbname]
user = self.get("db", "sql_user")
host = self.get("db", "sql_host")
port = self.get("db", "port")
pw = self.get("passwords", "sql")
#form the dsn (from components we have...)
#dsn = "<driver>://<username>:<password>@<host>:<port>/<database>"
if not host:
msg = "Unable to fetch a database dsn - no sql host found"
raise excp.BadParamException(msg)
driver = self.get("db", "type")
if not driver:
msg = "Unable to fetch a database dsn - no db driver type found"
raise excp.BadParamException(msg)
dsn = driver + "://"
if user:
dsn += user
if pw:
dsn += ":" + pw
if user or pw:
dsn += "@"
dsn += host
if port:
dsn += ":" + port
if dbname:
dsn += "/" + dbname
else:
dsn += "/"
LOG.debug("For database [%s] fetched dsn [%s] %s" % (dbname, dsn, CACHE_MSG))
#store for later...
self.db_dsns[dbname] = dsn
return dsn
def add_header(fn, contents): def add_header(fn, contents):
lines = list() lines = list()

69
devstack/cfg_helpers.py Normal file
View File

@ -0,0 +1,69 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from devstack import exceptions as excp
from devstack import log as logging
from devstack import settings
LOG = logging.getLogger("devstack.cfg.helpers")
def make_id(section, option):
joinwhat = []
if section is not None:
joinwhat.append(str(section))
if option is not None:
joinwhat.append(str(option))
return "/".join(joinwhat)
def fetch_run_type(config):
run_type = config.getdefaulted("default", "run_type", settings.RUN_TYPE_DEF)
run_type = run_type.upper()
return run_type
def fetch_dbdsn(config, pw_gen, dbname=''):
#check the dsn cache
user = config.get("db", "sql_user")
host = config.get("db", "sql_host")
port = config.get("db", "port")
pw = pw_gen.get_password("sql")
#form the dsn (from components we have...)
#dsn = "<driver>://<username>:<password>@<host>:<port>/<database>"
if not host:
msg = "Unable to fetch a database dsn - no sql host found"
raise excp.BadParamException(msg)
driver = config.get("db", "type")
if not driver:
msg = "Unable to fetch a database dsn - no db driver type found"
raise excp.BadParamException(msg)
dsn = driver + "://"
if user:
dsn += user
if pw:
dsn += ":" + pw
if user or pw:
dsn += "@"
dsn += host
if port:
dsn += ":" + port
if dbname:
dsn += "/" + dbname
else:
dsn += "/"
LOG.debug("For database [%s] fetched dsn [%s]" % (dbname, dsn))
return dsn

25
devstack/component.py
View File

@ -16,7 +16,7 @@
import weakref import weakref
from devstack import cfg from devstack import cfg_helpers
from devstack import downloader as down from devstack import downloader as down
from devstack import exceptions as excp from devstack import exceptions as excp
from devstack import log as logging from devstack import log as logging
@ -53,19 +53,22 @@ BASE_LINK_DIR = "/etc"
class ComponentBase(object): class ComponentBase(object):
def __init__(self, component_name, runner, root, opts, instances=None, def __init__(self, component_name, runner,
root_dir, component_options,
instances=None,
**kwds): **kwds):
self.component_name = component_name self.component_name = component_name
# The runner has a reference to us, so use a weakref here to # The runner has a reference to us, so use a weakref here to
# avoid breaking garbage collection. # avoid breaking garbage collection.
self.runner = weakref.proxy(runner) self.runner = weakref.proxy(runner)
self.root = root self.root = root_dir
self.component_opts = opts or [] self.component_opts = component_options or []
self.instances = instances or {} self.instances = instances or {}
# Parts of the global runner context that we use # Parts of the global runner context that we use
self.cfg = runner.cfg self.cfg = runner.cfg
self.password_generator = runner.password_generator self.pw_gen = runner.pw_gen
self.packager = runner.pkg_manager self.packager = runner.pkg_manager
self.distro = runner.distro self.distro = runner.distro
@ -79,9 +82,7 @@ class ComponentBase(object):
self.kargs = kwds self.kargs = kwds
def get_dependencies(self): def get_dependencies(self):
deps = settings.COMPONENT_DEPENDENCIES.get(self.component_name) deps = settings.COMPONENT_DEPENDENCIES.get(self.component_name) or list()
if not deps:
return list()
return list(deps) return list(deps)
def verify(self): def verify(self):
@ -124,13 +125,13 @@ class PkgInstallComponent(ComponentBase):
branch = self.cfg.get(cfg_section, cfg_key) branch = self.cfg.get(cfg_section, cfg_key)
if not branch: if not branch:
msg = "No branch entry found at config location [%s]" % \ msg = "No branch entry found at config location [%s]" % \
(cfg.make_id(cfg_section, cfg_key)) (cfg_helpers.make_id(cfg_section, cfg_key))
raise excp.ConfigException(msg) raise excp.ConfigException(msg)
(cfg_section, cfg_key) = uri_tuple (cfg_section, cfg_key) = uri_tuple
uri = self.cfg.get(cfg_section, cfg_key) uri = self.cfg.get(cfg_section, cfg_key)
if not uri: if not uri:
msg = "No uri entry found at config location [%s]" % \ msg = "No uri entry found at config location [%s]" % \
(cfg.make_id(cfg_section, cfg_key)) (cfg_helpers.make_id(cfg_section, cfg_key))
raise excp.ConfigException(msg) raise excp.ConfigException(msg)
self.tracewriter.download_happened(target_loc, uri) self.tracewriter.download_happened(target_loc, uri)
dirs_made = down.download(target_loc, uri, branch) dirs_made = down.download(target_loc, uri, branch)
@ -450,7 +451,7 @@ class ProgramRuntime(ComponentBase):
def configure(self): def configure(self):
# First make a pass and make sure all runtime (e.g. upstart) # First make a pass and make sure all runtime (e.g. upstart)
# config files are in place.... # config files are in place....
cls = RUNNER_CLS_MAPPING[utils.fetch_run_type(self.cfg)] cls = RUNNER_CLS_MAPPING[cfg_helpers.fetch_run_type(self.cfg)]
instance = cls(self.cfg, self.component_name, self.tracedir) instance = cls(self.cfg, self.component_name, self.tracedir)
tot_am = 0 tot_am = 0
for app_info in self._get_apps_to_start(): for app_info in self._get_apps_to_start():
@ -473,7 +474,7 @@ class ProgramRuntime(ComponentBase):
def start(self): def start(self):
# Select how we are going to start it # Select how we are going to start it
cls = RUNNER_CLS_MAPPING[utils.fetch_run_type(self.cfg)] cls = RUNNER_CLS_MAPPING[cfg_helpers.fetch_run_type(self.cfg)]
instance = cls(self.cfg, self.component_name, self.tracedir) instance = cls(self.cfg, self.component_name, self.tracedir)
am_started = 0 am_started = 0
for app_info in self._get_apps_to_start(): for app_info in self._get_apps_to_start():

30
devstack/components/db.py
View File

@ -85,9 +85,6 @@ REQ_PKGS = ['db.json']
#config keys we warm up so u won't be prompted later #config keys we warm up so u won't be prompted later
WARMUP_PWS = ['sql'] WARMUP_PWS = ['sql']
#partial of database user prompt
PASSWORD_DESCRIPTION = 'the database user'
class DBUninstaller(comp.PkgUninstallComponent): class DBUninstaller(comp.PkgUninstallComponent):
def __init__(self, *args, **kargs): def __init__(self, *args, **kargs):
@ -96,8 +93,7 @@ class DBUninstaller(comp.PkgUninstallComponent):
def warm_configs(self): def warm_configs(self):
for pw_key in WARMUP_PWS: for pw_key in WARMUP_PWS:
self.password_generator.get_password( self.pw_gen.get_password(pw_key)
'passwords', pw_key, PASSWORD_DESCRIPTION)
def pre_uninstall(self): def pre_uninstall(self):
dbtype = self.cfg.get("db", "type") dbtype = self.cfg.get("db", "type")
@ -110,10 +106,8 @@ class DBUninstaller(comp.PkgUninstallComponent):
if pwd_cmd: if pwd_cmd:
LOG.info("Ensuring your database is started before we operate on it.") LOG.info("Ensuring your database is started before we operate on it.")
self.runtime.restart() self.runtime.restart()
old_pw = self.password_generator.get_password(
'passwords', 'sql', PASSWORD_DESCRIPTION)
params = { params = {
'OLD_PASSWORD': old_pw, 'OLD_PASSWORD': self.pw_gen.get_password('sql'),
'NEW_PASSWORD': RESET_BASE_PW, 'NEW_PASSWORD': RESET_BASE_PW,
'USER': self.cfg.getdefaulted("db", "sql_user", 'root'), 'USER': self.cfg.getdefaulted("db", "sql_user", 'root'),
} }
@ -135,8 +129,7 @@ class DBInstaller(comp.PkgInstallComponent):
#in pre-install and post-install sections #in pre-install and post-install sections
host_ip = self.cfg.get('host', 'ip') host_ip = self.cfg.get('host', 'ip')
out = { out = {
'PASSWORD': self.password_generator.get_password( 'PASSWORD': self.pw_gen.get_password("sql"),
"passwords", "sql", PASSWORD_DESCRIPTION),
'BOOT_START': ("%s" % (True)).lower(), 'BOOT_START': ("%s" % (True)).lower(),
'USER': self.cfg.getdefaulted("db", "sql_user", 'root'), 'USER': self.cfg.getdefaulted("db", "sql_user", 'root'),
'SERVICE_HOST': host_ip, 'SERVICE_HOST': host_ip,
@ -146,8 +139,7 @@ class DBInstaller(comp.PkgInstallComponent):
def warm_configs(self): def warm_configs(self):
for pw_key in WARMUP_PWS: for pw_key in WARMUP_PWS:
self.password_generator.get_password( self.pw_gen.get_password(pw_key)
'passwords', pw_key, PASSWORD_DESCRIPTION)
def _configure_db_confs(self): def _configure_db_confs(self):
dbtype = self.cfg.get("db", "type") dbtype = self.cfg.get("db", "type")
@ -200,8 +192,7 @@ class DBInstaller(comp.PkgInstallComponent):
LOG.info("Ensuring your database is started before we operate on it.") LOG.info("Ensuring your database is started before we operate on it.")
self.runtime.restart() self.runtime.restart()
params = { params = {
'NEW_PASSWORD': self.password_generator.get_password( 'NEW_PASSWORD': self.pw_gen.get_password("sql"),
"passwords", "sql", PASSWORD_DESCRIPTION),
'USER': self.cfg.getdefaulted("db", "sql_user", 'root'), 'USER': self.cfg.getdefaulted("db", "sql_user", 'root'),
'OLD_PASSWORD': RESET_BASE_PW, 'OLD_PASSWORD': RESET_BASE_PW,
} }
@ -220,8 +211,7 @@ class DBInstaller(comp.PkgInstallComponent):
LOG.info("Ensuring your database is started before we operate on it.") LOG.info("Ensuring your database is started before we operate on it.")
self.runtime.restart() self.runtime.restart()
params = { params = {
'PASSWORD': self.password_generator.get_password( 'PASSWORD': self.pw_gen.get_password("sql"),
"passwords", "sql", PASSWORD_DESCRIPTION),
'USER': user, 'USER': user,
} }
cmds = [{'cmd': grant_cmd}] cmds = [{'cmd': grant_cmd}]
@ -296,13 +286,13 @@ class DBRuntime(comp.EmptyRuntime):
return comp.STATUS_UNKNOWN return comp.STATUS_UNKNOWN
def drop_db(cfg, dbname): def drop_db(cfg, pw_gen, dbname):
dbtype = cfg.get("db", "type") dbtype = cfg.get("db", "type")
dbactions = DB_ACTIONS.get(dbtype) dbactions = DB_ACTIONS.get(dbtype)
if dbactions and dbactions.get('drop_db'): if dbactions and dbactions.get('drop_db'):
dropcmd = dbactions.get('drop_db') dropcmd = dbactions.get('drop_db')
params = dict() params = dict()
params['PASSWORD'] = cfg.get("passwords", "sql") params['PASSWORD'] = pw_gen.get_password("sql")
params['USER'] = cfg.getdefaulted("db", "sql_user", 'root') params['USER'] = cfg.getdefaulted("db", "sql_user", 'root')
params['DB'] = dbname params['DB'] = dbname
cmds = list() cmds = list()
@ -316,13 +306,13 @@ def drop_db(cfg, dbname):
raise NotImplementedError(msg) raise NotImplementedError(msg)
def create_db(cfg, dbname): def create_db(cfg, pw_gen, dbname):
dbtype = cfg.get("db", "type") dbtype = cfg.get("db", "type")
dbactions = DB_ACTIONS.get(dbtype) dbactions = DB_ACTIONS.get(dbtype)
if dbactions and dbactions.get('create_db'): if dbactions and dbactions.get('create_db'):
createcmd = dbactions.get('create_db') createcmd = dbactions.get('create_db')
params = dict() params = dict()
params['PASSWORD'] = cfg.get("passwords", "sql") params['PASSWORD'] = pw_gen.get_password("sql")
params['USER'] = cfg.getdefaulted("db", "sql_user", 'root') params['USER'] = cfg.getdefaulted("db", "sql_user", 'root')
params['DB'] = dbname params['DB'] = dbname
cmds = list() cmds = list()

11
devstack/components/glance.py
View File

@ -17,6 +17,7 @@
import io import io
from devstack import cfg from devstack import cfg
from devstack import cfg_helpers
from devstack import component as comp from devstack import component as comp
from devstack import log as logging from devstack import log as logging
from devstack import settings from devstack import settings
@ -118,8 +119,8 @@ class GlanceInstaller(comp.PythonInstallComponent):
def _setup_db(self): def _setup_db(self):
LOG.info("Fixing up database named %s.", DB_NAME) LOG.info("Fixing up database named %s.", DB_NAME)
db.drop_db(self.cfg, DB_NAME) db.drop_db(self.cfg, self.pw_gen, DB_NAME)
db.create_db(self.cfg, DB_NAME) db.create_db(self.cfg, self.pw_gen, DB_NAME)
def _get_source_config(self, config_fn): def _get_source_config(self, config_fn):
if config_fn == POLICY_JSON: if config_fn == POLICY_JSON:
@ -184,10 +185,10 @@ class GlanceInstaller(comp.PythonInstallComponent):
mp = dict() mp = dict()
mp['DEST'] = self.appdir mp['DEST'] = self.appdir
mp['SYSLOG'] = self.cfg.getboolean("default", "syslog") mp['SYSLOG'] = self.cfg.getboolean("default", "syslog")
mp['SQL_CONN'] = self.cfg.get_dbdsn(DB_NAME) mp['SQL_CONN'] = cfg_helpers.fetch_dbdsn(self.cfg, self.pw_gen, DB_NAME)
mp['SERVICE_HOST'] = self.cfg.get('host', 'ip') mp['SERVICE_HOST'] = self.cfg.get('host', 'ip')
mp['HOST_IP'] = self.cfg.get('host', 'ip') mp['HOST_IP'] = self.cfg.get('host', 'ip')
mp.update(keystone.get_shared_params(self.cfg, self.password_generator, 'glance')) mp.update(keystone.get_shared_params(self.cfg, self.pw_gen, 'glance'))
return mp return mp
@ -224,4 +225,4 @@ class GlanceRuntime(comp.PythonRuntime):
# TODO: make this less cheesy - need to wait till glance goes online # TODO: make this less cheesy - need to wait till glance goes online
LOG.info("Waiting %s seconds so that glance can start up before image install." % (WAIT_ONLINE_TO)) LOG.info("Waiting %s seconds so that glance can start up before image install." % (WAIT_ONLINE_TO))
sh.sleep(WAIT_ONLINE_TO) sh.sleep(WAIT_ONLINE_TO)
creator.ImageCreationService(self.cfg, self.password_generator).install() creator.ImageCreationService(self.cfg, self.pw_gen).install()

25
devstack/components/keystone.py
View File

@ -19,6 +19,7 @@ import io
from urlparse import urlunparse from urlparse import urlunparse
from devstack import cfg from devstack import cfg
from devstack import cfg_helpers
from devstack import component as comp from devstack import component as comp
from devstack import log as logging from devstack import log as logging
from devstack import settings from devstack import settings
@ -132,8 +133,8 @@ class KeystoneInstaller(comp.PythonInstallComponent):
def _setup_db(self): def _setup_db(self):
LOG.info("Fixing up database named %s.", DB_NAME) LOG.info("Fixing up database named %s.", DB_NAME)
db.drop_db(self.cfg, DB_NAME) db.drop_db(self.cfg, self.pw_gen, DB_NAME)
db.create_db(self.cfg, DB_NAME) db.create_db(self.cfg, self.pw_gen, DB_NAME)
def _setup_initer(self): def _setup_initer(self):
LOG.info("Configuring keystone initializer template %s.", MANAGE_DATA_CONF) LOG.info("Configuring keystone initializer template %s.", MANAGE_DATA_CONF)
@ -191,7 +192,7 @@ class KeystoneInstaller(comp.PythonInstallComponent):
return comp.PythonInstallComponent._get_source_config(self, config_fn) return comp.PythonInstallComponent._get_source_config(self, config_fn)
def warm_configs(self): def warm_configs(self):
get_shared_params(self.cfg, self.password_generator) get_shared_params(self.cfg, self.pw_gen)
def _get_param_map(self, config_fn): def _get_param_map(self, config_fn):
#these be used to fill in the configuration/cmds + #these be used to fill in the configuration/cmds +
@ -202,11 +203,11 @@ class KeystoneInstaller(comp.PythonInstallComponent):
mp['BIN_DIR'] = self.bindir mp['BIN_DIR'] = self.bindir
mp['CONFIG_FILE'] = sh.joinpths(self.cfgdir, ROOT_CONF) mp['CONFIG_FILE'] = sh.joinpths(self.cfgdir, ROOT_CONF)
if config_fn == ROOT_CONF: if config_fn == ROOT_CONF:
mp['SQL_CONN'] = self.cfg.get_dbdsn(DB_NAME) mp['SQL_CONN'] = cfg_helpers.fetch_dbdsn(self.cfg, self.pw_gen, DB_NAME)
mp['KEYSTONE_DIR'] = self.appdir mp['KEYSTONE_DIR'] = self.appdir
mp.update(get_shared_params(self.cfg, self.password_generator)) mp.update(get_shared_params(self.cfg, self.pw_gen))
elif config_fn == MANAGE_DATA_CONF: elif config_fn == MANAGE_DATA_CONF:
mp.update(get_shared_params(self.cfg, self.password_generator)) mp.update(get_shared_params(self.cfg, self.pw_gen))
return mp return mp
@ -248,8 +249,7 @@ class KeystoneRuntime(comp.PythonRuntime):
return APP_OPTIONS.get(app) return APP_OPTIONS.get(app)
def get_shared_params(config, password_generator, service_user_name=None): def get_shared_params(config, pw_gen, service_user_name=None):
LOG.debug('password_generator %s', password_generator)
mp = dict() mp = dict()
host_ip = config.get('host', 'ip') host_ip = config.get('host', 'ip')
@ -263,18 +263,15 @@ def get_shared_params(config, password_generator, service_user_name=None):
mp['DEMO_TENANT_NAME'] = mp['DEMO_USER_NAME'] mp['DEMO_TENANT_NAME'] = mp['DEMO_USER_NAME']
#tokens and passwords #tokens and passwords
mp['SERVICE_TOKEN'] = password_generator.get_password( mp['SERVICE_TOKEN'] = pw_gen.get_password(
'passwords',
"service_token", "service_token",
'the service admin token', 'the service admin token',
) )
mp['ADMIN_PASSWORD'] = password_generator.get_password( mp['ADMIN_PASSWORD'] = pw_gen.get_password(
'passwords',
'horizon_keystone_admin', 'horizon_keystone_admin',
'the horizon and keystone admin', 'the horizon and keystone admin',
20) 20)
mp['SERVICE_PASSWORD'] = password_generator.get_password( mp['SERVICE_PASSWORD'] = pw_gen.get_password(
'passwords',
'service_password', 'service_password',
'service authentication', 'service authentication',
) )

7
devstack/components/melange.py
View File

@ -17,6 +17,7 @@
import io import io
from devstack import cfg from devstack import cfg
from devstack import cfg_helpers
from devstack import component as comp from devstack import component as comp
from devstack import log as logging from devstack import log as logging
from devstack import settings from devstack import settings
@ -88,8 +89,8 @@ class MelangeInstaller(comp.PythonInstallComponent):
def _setup_db(self): def _setup_db(self):
LOG.info("Fixing up database named %s.", DB_NAME) LOG.info("Fixing up database named %s.", DB_NAME)
db.drop_db(self.cfg, DB_NAME) db.drop_db(self.cfg, self.pw_gen, DB_NAME)
db.create_db(self.cfg, DB_NAME) db.create_db(self.cfg, self.pw_gen, DB_NAME)
def _get_pkgs(self): def _get_pkgs(self):
return list(REQ_PKGS) return list(REQ_PKGS)
@ -115,7 +116,7 @@ class MelangeInstaller(comp.PythonInstallComponent):
with io.BytesIO(contents) as stream: with io.BytesIO(contents) as stream:
config = cfg.IgnoreMissingConfigParser() config = cfg.IgnoreMissingConfigParser()
config.readfp(stream) config.readfp(stream)
db_dsn = self.cfg.get_dbdsn(DB_NAME) db_dsn = cfg_helpers.fetch_dbdsn(self.cfg, self.pw_gen, DB_NAME)
old_dbsn = config.get('DEFAULT', 'sql_connection') old_dbsn = config.get('DEFAULT', 'sql_connection')
if db_dsn != old_dbsn: if db_dsn != old_dbsn:
config.set('DEFAULT', 'sql_connection', db_dsn) config.set('DEFAULT', 'sql_connection', db_dsn)

11
devstack/components/nova.py
View File

@ -16,6 +16,7 @@
from urlparse import urlunparse from urlparse import urlunparse
from devstack import cfg_helpers
from devstack import component as comp from devstack import component as comp
from devstack import date from devstack import date
from devstack import exceptions from devstack import exceptions
@ -371,8 +372,8 @@ class NovaInstaller(comp.PythonInstallComponent):
def _setup_db(self): def _setup_db(self):
LOG.info("Fixing up database named %s.", DB_NAME) LOG.info("Fixing up database named %s.", DB_NAME)
db.drop_db(self.cfg, DB_NAME) db.drop_db(self.cfg, self.pw_gen, DB_NAME)
db.create_db(self.cfg, DB_NAME) db.create_db(self.cfg, self.pw_gen, DB_NAME)
def _generate_nova_conf(self): def _generate_nova_conf(self):
LOG.info("Generating dynamic content for nova configuration (%s)." % (API_CONF)) LOG.info("Generating dynamic content for nova configuration (%s)." % (API_CONF))
@ -406,7 +407,7 @@ class NovaInstaller(comp.PythonInstallComponent):
mp['FIXED_NETWORK_SIZE'] = self.cfg.getdefaulted('nova', 'fixed_network_size', '256') mp['FIXED_NETWORK_SIZE'] = self.cfg.getdefaulted('nova', 'fixed_network_size', '256')
mp['FIXED_RANGE'] = self.cfg.getdefaulted('nova', 'fixed_range', '10.0.0.0/24') mp['FIXED_RANGE'] = self.cfg.getdefaulted('nova', 'fixed_range', '10.0.0.0/24')
else: else:
mp.update(keystone.get_shared_params(self.cfg, self.password_generator, 'nova')) mp.update(keystone.get_shared_params(self.cfg, self.pw_gen, 'nova'))
return mp return mp
def configure(self): def configure(self):
@ -574,6 +575,7 @@ class NovaVolumeConfigurator(object):
class NovaConfConfigurator(object): class NovaConfConfigurator(object):
def __init__(self, ni): def __init__(self, ni):
self.cfg = ni.cfg self.cfg = ni.cfg
self.pw_gen = ni.pw_gen
self.instances = ni.instances self.instances = ni.instances
self.component_root = ni.component_root self.component_root = ni.component_root
self.appdir = ni.appdir self.appdir = ni.appdir
@ -637,7 +639,8 @@ class NovaConfConfigurator(object):
nova_conf.add('my_ip', hostip) nova_conf.add('my_ip', hostip)
#setup your sql connection #setup your sql connection
nova_conf.add('sql_connection', self.cfg.get_dbdsn('nova')) db_dsn = cfg_helpers.fetch_dbdsn(self.cfg, self.pw_gen, DB_NAME)
nova_conf.add('sql_connection', db_dsn)
#configure anything libvirt releated? #configure anything libvirt releated?
virt_driver = _canon_virt_driver(self._getstr('virt_driver')) virt_driver = _canon_virt_driver(self._getstr('virt_driver'))

7
devstack/components/quantum.py
View File

@ -17,6 +17,7 @@
import io import io
from devstack import cfg from devstack import cfg
from devstack import cfg_helpers
from devstack import component as comp from devstack import component as comp
from devstack import log as logging from devstack import log as logging
from devstack import settings from devstack import settings
@ -162,7 +163,7 @@ class QuantumInstaller(comp.PkgInstallComponent):
config.readfp(stream) config.readfp(stream)
db_dsn = config.get("DATABASE", "sql_connection") db_dsn = config.get("DATABASE", "sql_connection")
if db_dsn: if db_dsn:
generated_dsn = self.cfg.get_dbdsn(DB_NAME) generated_dsn = cfg_helpers.fetch_dbdsn(self.cfg, self.pw_gen, DB_NAME)
if generated_dsn != db_dsn: if generated_dsn != db_dsn:
config.set("DATABASE", "sql_connection", generated_dsn) config.set("DATABASE", "sql_connection", generated_dsn)
with io.BytesIO() as outputstream: with io.BytesIO() as outputstream:
@ -199,8 +200,8 @@ class QuantumInstaller(comp.PkgInstallComponent):
def _setup_db(self): def _setup_db(self):
LOG.info("Fixing up database named %s.", DB_NAME) LOG.info("Fixing up database named %s.", DB_NAME)
db.drop_db(self.cfg, DB_NAME) db.drop_db(self.cfg, self.pw_gen, DB_NAME)
db.create_db(self.cfg, DB_NAME) db.create_db(self.cfg, self.pw_gen, DB_NAME)
def _get_source_config(self, config_fn): def _get_source_config(self, config_fn):
if config_fn == PLUGIN_CONF: if config_fn == PLUGIN_CONF:

4
devstack/components/rabbit.py
View File

@ -71,12 +71,12 @@ class RabbitInstaller(comp.PkgInstallComponent):
def warm_configs(self): def warm_configs(self):
for pw_key in WARMUP_PWS: for pw_key in WARMUP_PWS:
self.password_generator.get_password("passwords", pw_key, PW_USER_PROMPT) self.pw_gen.get_password(pw_key, PW_USER_PROMPT)
def _setup_pw(self): def _setup_pw(self):
LOG.info("Setting up your rabbit-mq guest password.") LOG.info("Setting up your rabbit-mq guest password.")
self.runtime.restart() self.runtime.restart()
passwd = self.password_generator.get_password('passwords', "rabbit", PW_USER_PROMPT) passwd = self.pw_gen.get_password("rabbit", PW_USER_PROMPT)
cmd = PWD_CMD + [passwd] cmd = PWD_CMD + [passwd]
sh.execute(*cmd, run_as_root=True) sh.execute(*cmd, run_as_root=True)
LOG.info("Restarting so that your rabbit-mq guest password is reflected.") LOG.info("Restarting so that your rabbit-mq guest password is reflected.")

6
devstack/env_rc.py
View File

@ -51,9 +51,9 @@ QUOTED_PAT = re.compile(r"^\s*[\"](.*)[\"]\s*$")
class RcWriter(object): class RcWriter(object):
def __init__(self, cfg, password_generator): def __init__(self, cfg, pw_gen):
self.cfg = cfg self.cfg = cfg
self.password_generator = password_generator self.pw_gen = pw_gen
def _make_export(self, export_name, value): def _make_export(self, export_name, value):
escaped_val = sh.shellquote(value) escaped_val = sh.shellquote(value)
@ -153,7 +153,7 @@ class RcWriter(object):
sh.write_file(fn, contents) sh.write_file(fn, contents)
def _get_os_envs(self): def _get_os_envs(self):
key_params = keystone.get_shared_params(self.cfg, self.password_generator) key_params = keystone.get_shared_params(self.cfg, self.pw_gen)
to_set = dict() to_set = dict()
to_set['OS_PASSWORD'] = key_params['ADMIN_PASSWORD'] to_set['OS_PASSWORD'] = key_params['ADMIN_PASSWORD']
to_set['OS_TENANT_NAME'] = key_params['DEMO_TENANT_NAME'] to_set['OS_TENANT_NAME'] = key_params['DEMO_TENANT_NAME']

6
devstack/image/creator.py
View File

@ -211,14 +211,14 @@ class ImageRegistry:
class ImageCreationService: class ImageCreationService:
def __init__(self, cfg, password_generator): def __init__(self, cfg, pw_gen):
self.cfg = cfg self.cfg = cfg
self.password_generator = password_generator self.pw_gen = pw_gen
def _get_token(self): def _get_token(self):
LOG.info("Fetching your keystone admin token so that we can perform image uploads.") LOG.info("Fetching your keystone admin token so that we can perform image uploads.")
key_params = keystone.get_shared_params(self.cfg, self.password_generator) key_params = keystone.get_shared_params(self.cfg, self.pw_gen)
keystone_service_url = key_params['SERVICE_ENDPOINT'] keystone_service_url = key_params['SERVICE_ENDPOINT']
keystone_token_url = "%s/tokens" % (keystone_service_url) keystone_token_url = "%s/tokens" % (keystone_service_url)

3
devstack/opts.py
View File

@ -53,7 +53,8 @@ def parse():
action="store_true", action="store_true",
dest="dryrun", dest="dryrun",
default=False, default=False,
help="perform actions without actually doing any of them and/or attempt to do this: (default: %default)") help=("perform ACTION but do not actually run any of the commands"
" that would normally complete ACTION: (default: %default)"))
base_group = OptionGroup(parser, "Install & uninstall & start & stop specific options") base_group = OptionGroup(parser, "Install & uninstall & start & stop specific options")
base_group.add_option("-a", "--action", base_group.add_option("-a", "--action",

45
devstack/passwords.py
View File

@ -7,9 +7,17 @@ import logging
import os import os
import re import re
from devstack.cfg import make_id from devstack import cfg_helpers
LOG = logging.getLogger("devstack.passwords") LOG = logging.getLogger("devstack.passwords")
PW_SECTION = 'passwords'
HELPFUL_DESCRIPTIONS = {
'sql': 'the database user',
}
def get_pw_usage(option):
return HELPFUL_DESCRIPTIONS.get(option, '???')
def generate_random(length): def generate_random(length):
@ -21,18 +29,16 @@ def generate_random(length):
class PasswordGenerator(object): class PasswordGenerator(object):
def __init__(self, cfg, prompt_user=True): def __init__(self, kv_cache, cfg,
prompt_user=True):
self.cfg = cfg self.cfg = cfg
self.config_cache = kv_cache
self.prompt_user = prompt_user self.prompt_user = prompt_user
# Store the values accessed by the caller
# so the main script can print them out
# at the end.
self.accessed = {}
def _prompt_user(self, prompt_text): def _prompt_user(self, prompt_text):
LOG.debug('Asking the user for a %r password', prompt_text) LOG.debug('Asking the user for a %r password', prompt_text)
message = ("Enter a password to use for %s " message = ("Enter a password to use for %s "
"[or press enter to get a generated one] " % prompt_text "[or press enter to get a generated one]: " % prompt_text
) )
rc = "" rc = ""
while True: while True:
@ -50,21 +56,27 @@ class PasswordGenerator(object):
break break
return rc return rc
# FIXME: Remove the "section" argument, since it is always the same. def get_password(self, option, prompt_text=None, length=8):
def get_password(self, section, option, prompt_text, length=8):
"""Returns a password identified by the configuration location.""" """Returns a password identified by the configuration location."""
if not prompt_text:
prompt_text = get_pw_usage(option)
LOG.debug('Looking for password %s (%s)', option, prompt_text) LOG.debug('Looking for password %s (%s)', option, prompt_text)
cache_key = cfg_helpers.make_id(PW_SECTION, option)
password = self.config_cache.get(cache_key)
# Look in the configuration file(s) # Look in the configuration file(s)
try: if not password:
password = self.cfg.get(section, option) try:
except ConfigParser.Error: password = self.cfg.get(PW_SECTION, option)
password = '' except ConfigParser.Error:
password = ''
# Optionally ask the user # Optionally ask the user
if not password and self.prompt_user: if not password and self.prompt_user:
password = self._prompt_user(prompt_text) password = self._prompt_user(prompt_text)
self.accessed[make_id(section, option)] = password
# If we still don't have a value, make one up. # If we still don't have a value, make one up.
if not password: if not password:
@ -72,7 +84,8 @@ class PasswordGenerator(object):
option, prompt_text) option, prompt_text)
password = generate_random(length) password = generate_random(length)
# Update the configration cache so that other parts of the # Update the cache so that other parts of the
# code can find the value. # code can find the value.
self.cfg.set(section, option, password) self.config_cache[cache_key] = password
return password return password

13
devstack/progs/actions.py
View File

@ -139,13 +139,13 @@ PREQ_ACTIONS = {
class ActionRunner(object): class ActionRunner(object):
def __init__(self, distro, action, directory, config, def __init__(self, distro, action, directory, config,
password_generator, pkg_manager, pw_gen, pkg_manager,
**kargs): **kargs):
self.distro = distro self.distro = distro
self.action = action self.action = action
self.directory = directory self.directory = directory
self.cfg = config self.cfg = config
self.password_generator = password_generator self.pw_gen = pw_gen
self.pkg_manager = pkg_manager self.pkg_manager = pkg_manager
self.kargs = kargs self.kargs = kargs
self.components = dict() self.components = dict()
@ -205,8 +205,8 @@ class ActionRunner(object):
# the component keep a weakref to it. # the component keep a weakref to it.
instance = cls(instances=all_instances, instance = cls(instances=all_instances,
runner=self, runner=self,
root=self.directory, root_dir=self.directory,
opts=components.get(component, list()), component_options=components.get(component),
keep_old=self.kargs.get("keep_old") keep_old=self.kargs.get("keep_old")
) )
all_instances[component] = instance all_instances[component] = instance
@ -225,7 +225,8 @@ class ActionRunner(object):
if preq_components: if preq_components:
LOG.info("Having to activate prerequisite action [%s] for %s components." % (preq_action, len(preq_components))) LOG.info("Having to activate prerequisite action [%s] for %s components." % (preq_action, len(preq_components)))
preq_runner = ActionRunner(self.distro, preq_action, preq_runner = ActionRunner(self.distro, preq_action,
self.directory, self.cfg, self.pkg_manager, self.directory, self.cfg, self.pw_gen,
self.pkg_manager,
components=preq_components, **self.kargs) components=preq_components, **self.kargs)
preq_runner.run() preq_runner.run()
@ -248,7 +249,7 @@ class ActionRunner(object):
inst = instances[component] inst = instances[component]
inst.warm_configs() inst.warm_configs()
if self.gen_rc and self.rc_file: if self.gen_rc and self.rc_file:
writer = env_rc.RcWriter(self.cfg, self.password_generator) writer = env_rc.RcWriter(self.cfg, self.pw_gen)
if not sh.isfile(self.rc_file): if not sh.isfile(self.rc_file):
LOG.info("Generating a file at [%s] that will contain your environment settings." % (self.rc_file)) LOG.info("Generating a file at [%s] that will contain your environment settings." % (self.rc_file))
writer.write(self.rc_file) writer.write(self.rc_file)

19
devstack/progs/common.py
View File

@ -155,10 +155,12 @@ def get_packager(distro, keep_packages):
return cls(distro, keep_packages) return cls(distro, keep_packages)
def get_config(cfg_fn=None): def get_config(cfg_fn=None, kv_cache=None):
if not cfg_fn: if not cfg_fn:
cfg_fn = sh.canon_path(settings.STACK_CONFIG_LOCATION) cfg_fn = sh.canon_path(settings.STACK_CONFIG_LOCATION)
config_instance = cfg.StackConfigParser() if kv_cache is None:
kv_cache = dict()
config_instance = cfg.StackConfigParser(kv_cache)
config_instance.read(cfg_fn) config_instance.read(cfg_fn)
return config_instance return config_instance
@ -174,16 +176,15 @@ def get_components_deps(runner,
root_dir = root_dir or _FAKE_ROOT_DIR root_dir = root_dir or _FAKE_ROOT_DIR
while len(active_names): while len(active_names):
component = active_names.pop() component = active_names.pop()
component_opts = base_components.get(component) or [] component_opts = base_components.get(component) or list()
cls = get_action_cls(action_name, component, distro) cls = get_action_cls(action_name, component, distro)
instance = cls(instances=[], instance = cls(instances=list(),
runner=runner, runner=runner,
root=root_dir, root_dir=root_dir,
opts=component_opts, component_options=component_opts,
keep_old=False
) )
deps = instance.get_dependencies() deps = instance.get_dependencies() or set()
if deps is None:
deps = set()
all_components[component] = set(deps) all_components[component] = set(deps)
for d in deps: for d in deps:
if d not in all_components and d not in active_names: if d not in all_components and d not in active_names:

6
devstack/utils.py
View File

@ -206,12 +206,6 @@ def get_host_ip():
return ip return ip
def fetch_run_type(config):
run_type = config.getdefaulted("default", "run_type", settings.RUN_TYPE_DEF)
run_type = run_type.upper()
return run_type
def is_interface(intfc): def is_interface(intfc):
if intfc in get_interfaces(): if intfc in get_interfaces():
return True return True

62
stack
View File

@ -20,6 +20,7 @@ import sys
import time import time
import traceback import traceback
from devstack import cfg_helpers
from devstack import colorlog from devstack import colorlog
from devstack import date from devstack import date
from devstack import log as logging from devstack import log as logging
@ -32,6 +33,7 @@ from devstack import utils
from devstack.progs import actions from devstack.progs import actions
from devstack.progs import common from devstack.progs import common
LOG = logging.getLogger("devstack.stack") LOG = logging.getLogger("devstack.stack")
# This is used to map an action to a useful string for # This is used to map an action to a useful string for
@ -43,8 +45,18 @@ _WELCOME_MAP = {
settings.STOP: "STOPPER", settings.STOP: "STOPPER",
} }
_CFG_GROUPS = {
cfg_helpers.make_id('passwords', None): 'Passwords',
cfg_helpers.make_id('db', None): 'Database info',
#catch all
cfg_helpers.make_id(None, None): 'Misc configs',
}
def dump_config(config_obj, password_generator): _CFG_ORDERING = sorted(_CFG_GROUPS.keys())
_CFG_ORDERING.reverse()
def dump_config(config_cache):
def item_format(key, value): def item_format(key, value):
return "\t%s=%s" % (str(key), str(value)) return "\t%s=%s" % (str(key), str(value))
@ -54,21 +66,26 @@ def dump_config(config_obj, password_generator):
value = mp.get(key) value = mp.get(key)
LOG.info(item_format(key, value)) LOG.info(item_format(key, value))
passwords_gotten = password_generator.accessed #first partition into our groups
full_cfgs = config_obj.configs_fetched partitions = dict()
db_dsns = config_obj.db_dsns for name in _CFG_ORDERING:
if passwords_gotten or full_cfgs or db_dsns: partitions[name] = dict()
if passwords_gotten:
LOG.info("Passwords:") #now put the config cached values into there partition
map_print(passwords_gotten) for (k, v) in config_cache.items():
if full_cfgs: for name in _CFG_ORDERING:
filtered = dict((k, v) for (k, v) in full_cfgs.items() if k not in passwords_gotten) entries = partitions[name]
if filtered: if k.startswith(name):
LOG.info("Configs:") entries[k] = v
map_print(filtered) break
if db_dsns:
LOG.info("Data source names:") #now print them..
map_print(db_dsns) for name in _CFG_ORDERING:
nice_name = _CFG_GROUPS.get(name)
LOG.info(nice_name + ":")
entries = partitions.get(name)
if entries:
map_print(entries)
def run(args): def run(args):
@ -91,20 +108,25 @@ def run(args):
#here on out we should be using the logger (and not print) #here on out we should be using the logger (and not print)
start_time = time.time() start_time = time.time()
config = common.get_config()
# Stash the dryrun value (if any) into the global configuration # Stash the dryrun value (if any) into the global configuration
sh.set_dryrun(args['dryrun']) sh.set_dryrun(args['dryrun'])
password_generator = passwords.PasswordGenerator(config, args['prompt_for_passwords'])
config_kv_cache = dict()
config = common.get_config(kv_cache=config_kv_cache)
pw_gen = passwords.PasswordGenerator(config_kv_cache, config, args['prompt_for_passwords'])
pkg_manager = common.get_packager(distro, args['keep_old']) pkg_manager = common.get_packager(distro, args['keep_old'])
components = utils.parse_components(args.pop("components")) components = utils.parse_components(args.pop("components"))
runner = actions.ActionRunner(distro, action, rootdir, config, password_generator, runner = actions.ActionRunner(distro, action, rootdir, config, pw_gen,
pkg_manager, components=components, **args) pkg_manager, components=components, **args)
LOG.info("Starting action [%s] on %s for distro [%s]" % (action, date.rcf8222date(), distro)) LOG.info("Starting action [%s] on %s for distro [%s]" % (action, date.rcf8222date(), distro))
runner.run() runner.run()
LOG.info("It took (%s) to complete action [%s]" % (common.format_secs_taken((time.time() - start_time)), action)) LOG.info("It took (%s) to complete action [%s]" % (common.format_secs_taken((time.time() - start_time)), action))
LOG.info("After action [%s] your settings which were created or read are:" % (action)) LOG.info("After action [%s] your settings which were created or read are:" % (action))
dump_config(config, password_generator)
dump_config(config_kv_cache)
return True return True