Use python keyring package and move rc files to /etc/anvil (the new master location)

1. Move the rc files to /etc/anvil as well
2. Add in usage of python keyring package
   and use it to store/fetch passwords instead
   of our own yaml password storage. This also
   allows for us to use a encrypted password
   storage (not turned on by default) if desired
   which is a nice to have.
3. Passwords are now stored in 'passwords.cfg'
   under /etc/anvil (which should be fine for
   now and can be relocated via a cli option
   if desired)

anvil/__main__.py

@@ -41,6 +41,7 @@ from anvil.pprint import center_text
 
 
 LOG = logging.getLogger()
+ANVIL_DIR = "/etc/anvil/"
 SETTINGS_FN = "/etc/anvil/settings.yaml"
 
 
@@ -85,6 +86,9 @@ def run(args):
     if 'dryrun' in args:
         env.set("ANVIL_DRYRUN", str(args['dryrun']))
 
+    # Ensure the anvil etc dir is there if others are about to use it
+    ensure_anvil_dir()
+
     # Load the distro
     dist = distro.load(settings.DISTRO_DIR)
     
@@ -100,7 +104,7 @@ def run(args):
     runner = runner_cls(distro=dist,
                         root_dir=root_dir,
                         name=action,
-                        **args)
+                        cli_opts=args)
 
     (repeat_string, line_max_len) = utils.welcome()
     print(center_text("Action Runner", repeat_string, line_max_len))
@@ -125,23 +129,27 @@ def run(args):
 
 def load_previous_settings():
     settings_prev = None
-    if sh.isfile(SETTINGS_FN):
-        try:
-            # Don't use sh here so that we always
-            # read this (even if dry-run)    
-            with open(SETTINGS_FN, 'r') as fh:
-                settings_prev = utils.load_yaml_text(fh.read())
-        except Exception:
-            pass
+    try:
+        # Don't use sh here so that we always
+        # read this (even if dry-run)    
+        with open(SETTINGS_FN, 'r') as fh:
+            settings_prev = utils.load_yaml_text(fh.read())
+    except Exception:
+        # Errors could be expected on format problems
+        # or on the file not being readable....
+        pass
     return settings_prev
 
 
+def ensure_anvil_dir():
+    if not sh.isdir(ANVIL_DIR):
+        with sh.Rooted(True):
+            os.makedirs(ANVIL_DIR)
+            (uid, gid) = sh.get_suids()
+            sh.chown_r(ANVIL_DIR, uid, gid)
+
+
 def store_current_settings(settings):
-    base_dir = sh.dirname(SETTINGS_FN)
-    if not sh.isdir(base_dir):
-        # Don't use sh here so that we always
-        # read this (even if dry-run)    
-        os.makedirs(base_dir)
     try:
         with sh.Rooted(True):
             with open(SETTINGS_FN, 'w') as fh:
@@ -149,9 +157,9 @@ def store_current_settings(settings):
                 fh.write(utils.add_header(SETTINGS_FN, utils.prettify_yaml(settings)))
                 fh.flush()
         (uid, gid) = sh.get_suids()
-        sh.chown_r(base_dir, uid, gid)
+        sh.chown(SETTINGS_FN, uid, gid)
     except Exception as e:
-        pass
+        LOG.debug("Failed writing to %s due to %s", SETTINGS_FN, e)
 
 
 def main():

anvil/action.py

@@ -43,33 +43,44 @@ class PhaseFunctors(object):
 class Action(object):
     __meta__ = abc.ABCMeta
 
-    def __init__(self, name, distro, root_dir, **kwargs):
+    def __init__(self, name, distro, root_dir, cli_opts):
         self.distro = distro
         self.root_dir = root_dir
         self.name = name
         self.interpolator = cfg.YamlInterpolator(settings.COMPONENT_CONF_DIR)
-        self.passwords = pw.ProxyPassword()
-        self.password_files = [
-            sh.joinpths(root_dir, 'passwords.yaml'),
-        ]
-        self.default_password_file = sh.joinpths(os.getcwd(), 'passwords.yaml')
-        self.password_files.append(self.default_password_file)
-        if kwargs.get('prompt_for_passwords'):
-            self.passwords.resolvers.append(pw.InputPassword())
-        self.passwords.resolvers.append(pw.RandomPassword())
-        self.store_passwords = kwargs.get('store_passwords')
-        self.kwargs = kwargs
+        self.passwords = {}
+        self.keyring_path = cli_opts.pop('keyring_path')
+        self.keyring_encrypted = cli_opts.pop('keyring_encrypted')
+        self.prompt_for_passwords = cli_opts.pop('prompt_for_passwords', False)
+        self.store_passwords = cli_opts.pop('store_passwords', True)
+        self.cli_opts = cli_opts # Stored for components to get any options
 
-    def _establish_passwords(self):
-        pw_read = []
-        for fn in self.password_files:
-            if sh.isfile(fn):
-                self.passwords.cache.update(utils.load_yaml(fn))
-                pw_read.append(fn)
-        if pw_read:
-            utils.log_iterable(pw_read,
-                               header="Updated passwords to be used from %s files" % len(pw_read),
-                               logger=LOG)
+    def _establish_passwords(self, component_order, instances):
+        kr = pw.KeyringProxy(self.keyring_path,
+                             self.keyring_encrypted,
+                             self.prompt_for_passwords,
+                             True)
+        LOG.info("Reading passwords using a %s", kr)
+        to_save = {}
+        self.passwords.clear()
+        already_gotten = set()
+        for c in component_order:
+            instance = instances[c]
+            wanted_passwords = instance.get_option('wanted_passwords') or []
+            if not wanted_passwords:
+                continue
+            for (name, prompt) in wanted_passwords.items():
+                if name in already_gotten:
+                    continue
+                (from_keyring, pw_provided) = kr.read(name, prompt)
+                if not from_keyring and self.store_passwords:
+                    to_save[name] = pw_provided
+                self.passwords[name] = pw_provided
+                already_gotten.add(name)
+        if to_save:
+            LOG.info("Saving %s passwords using a %s", len(to_save), kr)
+            for (name, pw_provided) in to_save.items():
+                kr.save(name, pw_provided)
 
     @abc.abstractproperty
     @property
@@ -112,30 +123,6 @@ class Action(object):
         opts.update(persona_opts)
         return opts
 
-    def _update_passwords(self):
-        if not self.store_passwords:
-            return
-        if not self.passwords.cache:
-            return
-        who_update = []
-        for fn in self.password_files:
-            if sh.isfile(fn):
-                who_update.append(fn)
-        if not who_update:
-            who_update.append(self.default_password_file)
-        who_done = []
-        for fn in who_update:
-            if sh.isfile(fn):
-                contents = utils.load_yaml(fn)
-            else:
-                contents = {}
-            contents.update(self.passwords.cache)
-            sh.write_file(fn, utils.add_header(fn, utils.prettify_yaml(contents)))
-            who_done.append(fn)
-        utils.log_iterable(who_done,
-                           header="Updated/created %s password files" % len(who_done),
-                           logger=LOG)
-
     def _merge_subsystems(self, component_subsys, desired_subsys):
         joined_subsys = {}
         if not component_subsys:
@@ -155,7 +142,7 @@ class Action(object):
             if action not in sibling_instances:
                 sibling_instances[action] = {}
             cls = importer.import_entry_point(cls_name)
-            sibling_params = utils.merge_dicts(params, self.kwargs, preserve=True)
+            sibling_params = utils.merge_dicts(params, self.cli_opts, preserve=True)
             sibling_params['instances'] = sibling_instances[action]
             LOG.debug("Construction of sibling component %r (%r) params are:", name, action)
             utils.log_object(sibling_params, logger=LOG, level=logging.DEBUG)
@@ -204,7 +191,7 @@ class Action(object):
             instance_params['subsystems'] = self._merge_subsystems((distro_opts.pop('subsystems', None) or {}),
                                                                    (persona_subsystems.get(c) or {}))
             instance_params['siblings'] = siblings
-            instance_params = utils.merge_dicts(instance_params, self.kwargs, preserve=True)
+            instance_params = utils.merge_dicts(instance_params, self.cli_opts, preserve=True)
             LOG.debug("Construction of %r params are:", c)
             utils.log_object(instance_params, logger=LOG, level=logging.DEBUG)
             instances[c] = cls(**instance_params)
@@ -224,12 +211,11 @@ class Action(object):
         LOG.info("Booting up your components.")
         LOG.debug("Starting environment settings:")
         utils.log_object(env.get(), logger=LOG, level=logging.DEBUG, item_max_len=64)
-        self._establish_passwords()
+        self._establish_passwords(component_order, instances)
         self._verify_components(component_order, instances)
         self._warm_components(component_order, instances)
-        self._update_passwords()
 
-    def _write_exports(self, component_order, instances, filename):
+    def _write_exports(self, component_order, instances, path):
         # TODO(harlowja) perhaps remove this since its only used in a subclass...
         pass
 
@@ -237,8 +223,8 @@ class Action(object):
         LOG.info("Tearing down your components.")
         LOG.debug("Final environment settings:")
         utils.log_object(env.get(), logger=LOG, level=logging.DEBUG, item_max_len=64)
-        self._write_exports(component_order, instances, filename="%s.rc" % (self.name))
-        self._update_passwords()
+        exports_filename = "%s.rc" % (self.name)
+        self._write_exports(component_order, instances, sh.joinpths("/etc/anvil", exports_filename))
 
     def _get_phase_directory(self, name=None):
         if not name:

anvil/actions/install.py

@@ -45,7 +45,7 @@ class InstallAction(action.Action):
     def lookup_name(self):
         return 'install'
 
-    def _write_exports(self, component_order, instances, filename):
+    def _write_exports(self, component_order, instances, path):
         entries = []
         contents = StringIO()
         contents.write("# Exports for action %s\n\n" % (self.name))
@@ -59,9 +59,9 @@ class InstallAction(action.Action):
                     contents.write("%s\n" % (export_entry))
                 contents.write("\n")
         if entries:
-            sh.write_file(filename, contents.getvalue())
+            sh.write_file(path, contents.getvalue())
             utils.log_iterable(entries,
-                               header="Wrote to %s %s exports" % (filename, len(entries)),
+                               header="Wrote to %s %s exports" % (path, len(entries)),
                                logger=LOG)
 
     def _run(self, persona, component_order, instances):

anvil/actions/status.py

@@ -35,9 +35,9 @@ STATUS_COLOR_MAP = {
 
 
 class StatusAction(action.Action):
-    def __init__(self, name, distro, root_dir, **kwargs):
-        action.Action.__init__(self, name, distro, root_dir, **kwargs)
-        self.show_amount = kwargs.get('show_amount')
+    def __init__(self, name, distro, root_dir, cli_opts):
+        action.Action.__init__(self, name, distro, root_dir, cli_opts)
+        self.show_amount = cli_opts.get('show_amount', 0)
 
     @property
     def lookup_name(self):

anvil/component.py

@@ -14,6 +14,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from anvil import exceptions as excp
 from anvil import log as logging
 from anvil import type_utils as tu
 from anvil import utils
@@ -48,8 +49,11 @@ class Component(object):
         # How we get any passwords we need
         self.passwords = passwords
 
-    def get_password(self, option, prompt_text, **kwargs):
-        return self.passwords.get_password(option, prompt_text, **kwargs)
+    def get_password(self, option):
+        pw_val = self.passwords.get(option)
+        if pw_val is None:
+            raise excp.PasswordException("Password asked for option %s but none was pre-populated!" % (option))
+        return pw_val
 
     def get_option(self, option, *options, **kwargs):
         option_value = utils.get_deep(self.options, [option] + list(options))

anvil/components/helpers/db.py

@@ -31,7 +31,7 @@ PASSWORD_PROMPT = 'the database user'
 
 def get_shared_passwords(component):
     mp = {}
-    mp['pw'] = component.get_password('sql', PASSWORD_PROMPT)
+    mp['pw'] = component.get_password('sql')
     return mp
 
 

anvil/components/helpers/keystone.py

@@ -134,20 +134,10 @@ class Initializer(object):
 
 def get_shared_passwords(component):
     mp = {}
-    mp['service_token'] = component.get_password(
-        "service_token",
-        'the service admin token',
-        )
-    mp['admin_password'] = component.get_password(
-        'horizon_keystone_admin',
-        'the horizon and keystone admin',
-        length=20,
-        )
-    mp['demo_password'] = mp['admin_password']
-    mp['service_password'] = component.get_password(
-        'service_password',
-        'service authentication',
-        )
+    mp['service_token'] = component.get_password("service_token")
+    mp['admin_password'] = component.get_password('admin_password')
+    mp['demo_password'] = component.get_password('demo_password')
+    mp['service_password'] = component.get_password('service_password')
     return mp
 
 

anvil/components/helpers/rabbit.py

@@ -19,11 +19,7 @@ from anvil import log
 
 LOG = log.getLogger(__name__)
 
-# Partial of rabbit user prompt
-PW_USER_PROMPT = 'the rabbit user'
-
 def get_shared_passwords(component):
     mp = {}
-    mp['pw'] = component.get_password('rabbit', PW_USER_PROMPT) 
+    mp['pw'] = component.get_password('rabbit')
     return mp
-

anvil/exceptions.py

@@ -63,6 +63,10 @@ class StatusException(AnvilException):
     pass
 
 
+class PasswordException(AnvilException):
+    pass
+
+
 class FileException(AnvilException):
     pass
 

anvil/opts.py

@@ -45,49 +45,58 @@ def parse(previous_settings=None):
 
     # Root options
     parser.add_option("-v", "--verbose",
-        action="store_true",
-        dest="verbose",
-        default=False,
-        help="make the output logging verbose")
+                      action="store_true",
+                      dest="verbose",
+                      default=False,
+                      help="make the output logging verbose")
     parser.add_option("--dryrun",
-        action="store_true",
-        dest="dryrun",
-        default=False,
-        help=("perform ACTION but do not actually run any of the commands"
-              " that would normally complete ACTION"))
+                      action="store_true",
+                      dest="dryrun",
+                      default=False,
+                      help=("perform ACTION but do not actually run any of the commands"
+                            " that would normally complete ACTION"))
+    parser.add_option('-k', "--keyring",
+                      action="store",
+                      dest="keyring_path",
+                      default="/etc/anvil/passwords.cfg",
+                      help=("read and create passwords using this keyring file (default: %default)"))
+    parser.add_option('-e', "--encrypt",
+                      action="store_true",
+                      dest="keyring_encrypted",
+                      default=False,
+                      help=("use a encrypted keyring file (default: %default)"))
+    parser.add_option("--no-prompt-passwords",
+                      action="store_false",
+                      dest="prompt_for_passwords",
+                      default=True,
+                      help="do not prompt the user for passwords")
+    parser.add_option("--no-store-passwords",
+                      action="store_false",
+                      dest="store_passwords",
+                      default=True,
+                      help="do not save the users passwords into the users keyring")
 
     # Install/start/stop/uninstall specific options
     base_group = OptionGroup(parser, "Action specific options")
     base_group.add_option("-p", "--persona",
-        action="store",
-        type="string",
-        dest="persona_fn",
-        default=sh.joinpths(settings.PERSONA_DIR, 'in-a-box', 'basic.yaml'),
-        metavar="FILE",
-        help="persona yaml file to apply (default: %default)")
+                          action="store",
+                          type="string",
+                          dest="persona_fn",
+                          default=sh.joinpths(settings.PERSONA_DIR, 'in-a-box', 'basic.yaml'),
+                          metavar="FILE",
+                          help="persona yaml file to apply (default: %default)")
     base_group.add_option("-a", "--action",
-        action="store",
-        type="string",
-        dest="action",
-        metavar="ACTION",
-        help="required action to perform: %s" % (_format_list(actions.names())))
+                          action="store",
+                          type="string",
+                          dest="action",
+                          metavar="ACTION",
+                          help="required action to perform: %s" % (_format_list(actions.names())))
     base_group.add_option("-d", "--directory",
-        action="store",
-        type="string",
-        dest="dir",
-        metavar="DIR",
-        help=("empty root DIR or "
-              "DIR with existing components"))
-    base_group.add_option("--no-prompt-passwords",
-                          action="store_false",
-                          dest="prompt_for_passwords",
-                          default=True,
-                          help="do not prompt the user for passwords")
-    base_group.add_option("--no-store-passwords",
-        action="store_false",
-        dest="store_passwords",
-        default=True,
-        help="do not store the users passwords into yaml files")
+                          action="store",
+                          type="string",
+                          dest="dir",
+                          metavar="DIR",
+                          help=("empty root DIR or DIR with existing components"))
     parser.add_option_group(base_group)
 
     suffixes = ("Known suffixes 'K' (kilobyte, 1024),"
@@ -95,28 +104,30 @@ def parse(previous_settings=None):
                 " are supported, 'B' is the default and is ignored")
     status_group = OptionGroup(parser, "Status specific options")
     status_group.add_option('-s', "--show",
-        action="callback",
-        dest="show_amount",
-        type='string',
-        metavar="SIZE",
-        callback=_size_cb,
-        help="show SIZE 'details' when showing component status. " + suffixes)
+                            action="callback",
+                            dest="show_amount",
+                            type='string',
+                            metavar="SIZE",
+                            callback=_size_cb,
+                            help="show SIZE 'details' when showing component status. " + suffixes)
     parser.add_option_group(status_group)
 
     pkg_group = OptionGroup(parser, "Packaging specific options")
     pkg_group.add_option('-m', "--match-installed",
-        action="store_true",
-        dest="match_installed",
-        default=False,
-        help="when packaging attempt to use the versions that are installed for the components dependencies")
+                         action="store_true",
+                         dest="match_installed",
+                         default=False,
+                         help=("when packaging attempt to use the versions that are "
+                               "installed for the components dependencies"))
     parser.add_option_group(pkg_group)
 
     uninstall_group = OptionGroup(parser, "Uninstall specific options")
     uninstall_group.add_option("--purge",
-        action="store_true",
-        dest="purge_packages",
-        default=False,
-        help=("assume when a package is not marked as removable that it can be removed (default: %default)"))
+                                action="store_true",
+                                dest="purge_packages",
+                                default=False,
+                                help=("assume when a package is not marked as"
+                                      " removable that it can be removed (default: %default)"))
     parser.add_option_group(uninstall_group)
 
     # Extract only what we care about, these will be passed
@@ -125,7 +136,7 @@ def parse(previous_settings=None):
     if previous_settings:
         parser.set_defaults(**previous_settings)
 
-    (options, args) = parser.parse_args()
+    (options, _args) = parser.parse_args()
     values = {}
     values['dir'] = (options.dir or "")
     values['dryrun'] = (options.dryrun or False)
@@ -137,4 +148,6 @@ def parse(previous_settings=None):
     values['store_passwords'] = options.store_passwords
     values['match_installed'] = options.match_installed
     values['purge_packages'] = options.purge_packages
+    values['keyring_path'] = options.keyring_path
+    values['keyring_encrypted'] = options.keyring_encrypted
     return values

anvil/passwords.py

@@ -19,41 +19,62 @@ import binascii
 import getpass
 import os
 
+from keyring.backend import CryptedFileKeyring
+from keyring.backend import UncryptedFileKeyring
+from keyring.util import properties
+
 from anvil import log as logging
+from anvil import shell as sh
+from anvil import utils
 
 LOG = logging.getLogger(__name__)
+RAND_PW_LEN = 20
+PW_USER = 'anvil'
 
+# There is some weird issue fixed after 0.9.2
+# this applies that fix for us for now (taken from the trunk code)...
+class FixedCryptedFileKeyring(CryptedFileKeyring):
 
-class ProxyPassword(object):
-    def __init__(self, cache=None):
-        if cache is None:
-            self.cache = {}
+    @properties.NonDataProperty
+    def keyring_key(self):
+        # _unlock or _init_file will set the key or raise an exception
+        if self._check_file():
+            self._unlock()
         else:
-            self.cache = cache
-        self.resolvers = []
+            self._init_file()
+        return self.keyring_key
 
-    def _valid_password(self, pw):
-        if pw is None:
-            return False
-        if len(pw) > 0:
-            return True
-        return False
 
-    def get_password(self, option, prompt_text='', length=8, **kwargs):
-        if option in self.cache:
-            return self.cache[option]
-        password = ''
-        for resolver in self.resolvers:
-            found_password = resolver.get_password(option,
-                                                   prompt_text=prompt_text,
-                                                   length=length, **kwargs)
-            if self._valid_password(found_password):
-                password = found_password
-                break
-        if len(password) == 0:
-            LOG.warn("Password provided for %r is empty", option)
-        self.cache[option] = password
-        return password
+class KeyringProxy(object):
+    def __init__(self, path, keyring_encrypted=False, enable_prompt=True, random_on_empty=True):
+        self.path = path
+        self.keyring_encrypted = keyring_encrypted
+        if keyring_encrypted:
+            self.ring = FixedCryptedFileKeyring()
+        else:
+            self.ring = UncryptedFileKeyring()
+        self.ring.file_path = path
+        self.enable_prompt = enable_prompt
+        self.random_on_empty = random_on_empty
+
+    def read(self, name, prompt):
+        pw_val = self.ring.get_password(name, PW_USER)
+        if pw_val:
+            return (True, pw_val)
+        if self.enable_prompt and prompt:
+            pw_val = InputPassword().get_password(name, prompt)
+        if self.random_on_empty and len(pw_val) == 0:
+            pw_val = RandomPassword().get_password(name, RAND_PW_LEN)
+        return (False, pw_val)
+    
+    def save(self, name, password):
+        self.ring.set_password(name, PW_USER, password)
+
+    def __str__(self):
+        prefix = 'encrypted'
+        if not self.keyring_encrypted:
+            prefix = "un" + prefix
+        return '%s keyring @ %s' % (prefix, self.path)
 
 
 class InputPassword(object):
@@ -65,8 +86,8 @@ class InputPassword(object):
             return True
 
     def _prompt_user(self, prompt_text):
-        LOG.debug('Asking the user for a %r password', prompt_text)
-        message = ("Enter a password to use for %s "
+        prompt_text = prompt_text.strip()
+        message = ("Enter a secret to use for the %s "
                    "[or press enter to get a generated one]: " % prompt_text
                    )
         rc = ""
@@ -79,8 +100,8 @@ class InputPassword(object):
                 LOG.warn("Invalid password %r (please try again)" % (rc))
         return rc
 
-    def get_password(self, option, **kargs):
-        return self._prompt_user(kargs.get('prompt_text', '??'))
+    def get_password(self, option, prompt_text):
+        return self._prompt_user(prompt_text)
 
 
 class RandomPassword(object):
@@ -92,5 +113,5 @@ class RandomPassword(object):
             return ''
         return binascii.hexlify(os.urandom((length + 1) / 2))[:length]
 
-    def get_password(self, option, **kargs):
-        return self.generate_random(int(kargs.get('length', 8)))
+    def get_password(self, option, length):
+        return self.generate_random(int(length))

conf/components/db.yaml

@@ -5,4 +5,7 @@ host: localhost
 port: 3306
 type: mysql
 user: root
+
+wanted_passwords:
+  sql: "database user"
 ...

conf/components/keystone.yaml

@@ -38,4 +38,10 @@ nova:
   ec2_admin_host: "$(nova:ec2_admin_host)"
   ec2_admin_port: "$(nova:ec2_admin_port)"
   protocol: "$(nova:protocol)"
+
+wanted_passwords:
+  service_token: 'service admin token'
+  admin_password: 'keystone admin user'
+  demo_password: 'keystone demo user'
+  service_password: 'service authentication password'
 ...

conf/components/rabbit-mq.yaml

@@ -5,4 +5,8 @@ host: "$(auto:ip)"
 
 # Which rabbit user should be used
 user_id: guest
+
+wanted_passwords:
+  rabbit: 'rabbit user'
+
 ...

smithy

@@ -74,25 +74,15 @@ EOF
         return 1
     fi
     echo "Installing needed pypi dependencies:"
-    pip-python install -U -I termcolor iniparse
+    pip-python install -U -I termcolor iniparse "keyring==0.9.2"
     if [ $? -ne 0 ]; then
         return 1
     fi
     return 0
 }
 
-load_rc_files()
-{
-    for i in `ls *.rc 2>/dev/null`; do
-        if [ -f "$i" ]; then
-            source "$i"
-        fi
-    done
-}
-
 run_smithy()
 {
-    load_rc_files
     PYTHON=`which python`
     exec $PYTHON anvil $ARGS
 }