Merge pull request #107 from harlowja/master

Adjusts to dougs comments

conf/personas/devstack.sh.yaml

@@ -1,10 +1,12 @@
 ---
-created_on: Wed, 14 Mar 2012 17:28:24 -0700
+description: Devstack.sh matching component installation.
-description: Devstack.sh matching component installation (as of the above date).
 supports:
 - rhel-6
 - ubuntu-oneiric
 - fedora-16
+options:
+  glance:
+    - load-images
 components:
 - db
 - rabbit-mq

devstack/component.py

@@ -93,10 +93,10 @@ class ComponentBase(object):
         knowns = self.known_subsystems()
         for s in self.desired_subsystems:
             if s not in knowns:
-                raise RuntimeError("Unknown subsystem %r requested" % (s))
+                raise ValueError("Unknown subsystem %r requested" % (s))
         for s in self.subsystem_info.keys():
             if s not in knowns:
-                raise RuntimeError("Unknown subsystem %r provided" % (s))
+                raise ValueError("Unknown subsystem %r provided" % (s))
 
     def known_subsystems(self):
         return list()

devstack/components/swift.py

@@ -62,7 +62,8 @@ CONFIG_DIR = 'etc'
 LOG_DIR = 'logs'
 
 # Config keys we warm up so u won't be prompted later
-WARMUP_PWS = ['service_token', 'swift_hash']
+WARMUP_PWS = [('service_token', 'the service admin token'),
+              ('swift_hash', 'the random unique string for your swift cluster')]
 
 
 class SwiftUninstaller(comp.PythonUninstallComponent):
@@ -107,8 +108,8 @@ class SwiftInstaller(comp.PythonInstallComponent):
         return list(CONFIGS)
 
     def warm_configs(self):
-        for pw_key in WARMUP_PWS:
+        for (pw_key, prompt) in WARMUP_PWS:
-            self.pw_gen.get_password(pw_key)
+            self.pw_gen.get_password(pw_key, prompt)
 
     def _get_param_map(self, config_fn):
         return {

devstack/distro.py

@@ -40,14 +40,22 @@ class Distro(object):
             raise RuntimeError(
                 'Did not find any distro definition files in %s' %
                 path)
-        for filename in input_files:
+        for fn in input_files:
+            cls_kvs = None
+            filename = sh.abspth(fn)
+            LOG.audit("Attempting to load distro definition from [%s]" % (filename))
             try:
                 with open(filename, 'r') as f:
-                    data = yaml.load(f)
+                    cls_kvs = yaml.load(f)
-                results.append(cls(**data))
             except (IOError, yaml.YAMLError) as err:
                 LOG.warning('Could not load distro definition from %s: %s',
                             filename, err)
+            if cls_kvs is not None:
+                try:
+                    results.append(cls(**cls_kvs))
+                except Exception as err:
+                    LOG.warning('Could not initialize instance %s using parameter map %s: %s',
+                                cls, cls_kvs, err)
         return results
 
     @classmethod
@@ -67,6 +75,7 @@ class Distro(object):
                 'No platform configuration data for %s (%s)' %
                 (plt, distname))
 
+    @logging.log_debug
     def __init__(self, name, distro_pattern, packager_name, commands, components):
         self.name = name
         self._distro_pattern = re.compile(distro_pattern, re.IGNORECASE)
@@ -74,9 +83,6 @@ class Distro(object):
         self._commands = commands
         self._components = components
 
-    def __repr__(self):
-        return "\"%s\" using packager \"%s\"" % (self.name, self._packager_name)
-
     def get_command(self, key, *more_keys, **kargs):
         """ Gets a end object for a given set of keys """
         root = self._commands

devstack/log.py

@@ -80,12 +80,10 @@ def getLogger(name='devstack'):
 
 def log_debug(f):
     @functools.wraps(f)
-    def wrapper(*args, **kw):
+    def wrapper(*args, **kargs):
-        if root.isEnabledFor(debug):
+        logger = getLogger()
-            logging.debug('%s(%s, %s) ->', f.func_name, str(args), str(kw))
+        logger.debug('%s(%s, %s) ->', f.func_name, str(args), str(kargs))
-        rv = f(*args, **kw)
+        rv = f(*args, **kargs)
-        if root.isEnabledFor(debug):
+        logger.debug("<- %s" % (pprint.pformat(rv, indent=2)))
-            logging.debug(pprint.pformat(rv, indent=2))
-            logging.debug('')
         return rv
     return wrapper

devstack/packager.py

@@ -21,6 +21,8 @@ LOG = logging.getLogger("devstack.packager")
 
 
 class Packager(object):
+
+    @logging.log_debug
     def __init__(self, distro, keep_packages):
         self.distro = distro
         self.keep_packages = keep_packages

devstack/persona.py

@@ -0,0 +1,86 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import yaml
+
+from devstack import exceptions as excp
+from devstack import log as logging
+from devstack import shell as sh
+
+LOG = logging.getLogger("devstack.persona")
+
+
+class Persona(object):
+
+    @classmethod
+    def load_file(cls, fn):
+        persona_fn = sh.abspth(fn)
+        LOG.audit("Loading persona from file [%s]", persona_fn)
+        cls_kvs = None
+        try:
+            with open(persona_fn, "r") as fh:
+                cls_kvs = yaml.load(fh.read())
+        except (IOError, yaml.YAMLError) as err:
+            LOG.warning('Could not load persona definition from %s: %s',
+                             persona_fn, err)
+        instance = None
+        if cls_kvs is not None:
+            try:
+                cls_kvs['source'] = persona_fn
+                instance = cls(**cls_kvs)
+            except Exception as err:
+                LOG.warning('Could not initialize instance %s using parameter map %s: %s',
+                                cls, cls_kvs, err)
+        return instance
+
+    @logging.log_debug
+    def __init__(self, description,
+                       supports,
+                       components,
+                       subsystems,
+                       options,
+                       **kargs):
+        self.distro_support = supports
+        self.wanted_components = components
+        self.source = kargs.get('source')  # May not always be there (ie if from a stream...)
+        self.wanted_subsystems = subsystems
+        self.description = description
+        self.component_options = options
+
+    def __str__(self):
+        info = "%s" % (self.description)
+        if self.source:
+            info += " from source %s:" % (self.source)
+        if self.wanted_subsystems:
+            info += " with desired subsystems (%s)" % (self.wanted_subsystems)
+        if self.wanted_components:
+            info += " with desired components (%s)" % (", ".join(self.wanted_components))
+        if self.component_options:
+            info += " with desired component options (%s)" % (", ".join(self.component_options))
+        if self.distro_support:
+            info += " which 'should' work on distros (%s)" % (", ".join(self.distro_support))
+        return info
+
+    def verify(self, distro):
+        # Some sanity checks against the given distro
+        d_name = distro.name
+        if d_name not in self.distro_support:
+            msg = "Distro %s not supported" % (d_name)
+            raise excp.ConfigException(msg)
+        for c in self.wanted_components:
+            if not distro.known_component(c):
+                raise RuntimeError("Distro %s does not support component %s" %
+                                        (d_name, c))

devstack/progs/actions.py

@@ -14,14 +14,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License..
 
-import yaml
-
 from devstack import env_rc
 from devstack import exceptions as excp
 from devstack import log as logging
 from devstack import settings
 from devstack import shell as sh
-from devstack import passwords
 
 LOG = logging.getLogger("devstack.progs.actions")
 
@@ -138,16 +135,16 @@ PREQ_ACTIONS = {
 
 
 class ActionRunner(object):
-    def __init__(self, distro, action, cfg, **kargs):
+    def __init__(self, distro, action,
+                    cfg, pw_gen, pkg_manager,
+                    **kargs):
         self.distro = distro
         self.action = action
         self.cfg = cfg
-        self.pw_gen = passwords.PasswordGenerator(self.cfg, kargs.get('prompt_for_passwords', True))
+        self.pw_gen = pw_gen
-        pkg_cls = distro.get_packager_factory()
+        self.pkg_manager = pkg_manager
-        self.keep_old = kargs.get('keep_old')
+        self.keep_old = kargs.get('keep_old', False)
-        self.pkg_manager = pkg_cls(self.distro, self.keep_old)
         self.force = kargs.get('force', False)
-        self.kargs = kargs
 
     def _apply_reverse(self, action, component_order):
         adjusted_order = list(component_order)
@@ -155,33 +152,9 @@ class ActionRunner(object):
             adjusted_order.reverse()
         return adjusted_order
 
-    def _load_persona(self, persona_fn):
-        persona_fn = sh.abspth(persona_fn)
-        LOG.audit("Loading persona from file [%s]", persona_fn)
-        contents = ''
-        with open(persona_fn, "r") as fh:
-            contents = fh.read()
-        return self._verify_persona(yaml.load(contents), persona_fn)
-
-    def _verify_persona(self, persona, fn):
-        # Some sanity checks
-        try:
-            if self.distro.name not in persona['supports']:
-                raise RuntimeError("Persona does not support distro %s"
-                                   % (self.distro.name))
-            for c in persona['components']:
-                if not self.distro.known_component(c):
-                    raise RuntimeError("Distro %s does not support component %s" %
-                                        (self.distro.name, c))
-        except (KeyError, RuntimeError) as e:
-            msg = ("Could not validate persona defined in [%s] due to: %s"
-                    % (fn, e))
-            raise excp.ConfigException(msg)
-        return persona
-
     def _construct_instances(self, persona, action, root_dir):
-        components = persona['components']  # Required
+        components = persona.wanted_components
-        desired_subsystems = persona.get('subsystems', dict())  # Not required
+        desired_subsystems = persona.wanted_subsystems or dict()
         instances = dict()
         for c in components:
             (cls, my_info) = self.distro.extract_component(c, action)
@@ -198,10 +171,7 @@ class ActionRunner(object):
             for (k, v) in my_info.items():
                 if k not in cls_kvs:
                     cls_kvs[k] = v
-            LOG.debug("Using arg map %s", cls_kvs)
+            instances[c] = cls(**cls_kvs)
-            cls_args = list()
-            LOG.debug("Using arg list %s", cls_args)
-            instances[c] = cls(*cls_args, **cls_kvs)
         return instances
 
     def _verify_components(self, component_order, instances):
@@ -243,7 +213,6 @@ class ActionRunner(object):
                         raise
 
     def _run_action(self, persona, action, root_dir):
-        LOG.info("Running action [%s] using root directory [%s]" % (action, root_dir))
         instances = self._construct_instances(persona, action, root_dir)
         if action in PREQ_ACTIONS:
             (check_functor, preq_action) = PREQ_ACTIONS[action]
@@ -255,7 +224,7 @@ class ActionRunner(object):
                 LOG.info("Activating prerequisite action [%s] requested by (%s) components."
                     % (preq_action, ", ".join(checks_passed_components)))
                 self._run_action(persona, preq_action, root_dir)
-        component_order = self._apply_reverse(action, persona['components'])
+        component_order = self._apply_reverse(action, persona.wanted_components)
         LOG.info("Activating components [%s] (in that order) for action [%s]" %
                   ("->".join(component_order), action))
         self._verify_components(component_order, instances)
@@ -264,11 +233,5 @@ class ActionRunner(object):
             self._write_rc_file(root_dir)
         self._run_instances(action, component_order, instances)
 
-    def _setup_root(self, root_dir):
+    def run(self, persona, root_dir):
-        if not sh.isdir(root_dir):
-            sh.mkdir(root_dir)
-
-    def run(self, persona_fn, root_dir):
-        persona = self._load_persona(persona_fn)
-        self._setup_root(root_dir)
         self._run_action(persona, self.action, root_dir)

stack

@@ -28,6 +28,8 @@ from devstack import env
 from devstack import env_rc
 from devstack import log as logging
 from devstack import opts
+from devstack import passwords
+from devstack import persona
 from devstack import settings
 from devstack import shell as sh
 from devstack import utils
@@ -101,6 +103,17 @@ def load_rc_files():
     return len(fns)
 
 
+def load_verify_persona(fn, dist):
+    instance = persona.Persona.load_file(fn)
+    instance.verify(dist)
+    return instance
+
+
+def setup_root(root_dir):
+    if not sh.isdir(root_dir):
+        sh.mkdir(root_dir)
+
+
 def run(args):
     action = args.pop("action", '').strip().lower()
     if not (action in settings.ACTIONS):
@@ -117,6 +130,7 @@ def run(args):
             print(utils.color_text("No root directory specified!", "red"))
             return False
     root_dir = sh.abspth(root_dir)
+    setup_root(root_dir)
 
     persona_fn = args.pop('persona_fn')
     if not persona_fn or not sh.isfile(persona_fn):
@@ -137,16 +151,27 @@ def run(args):
         loaded_rcs = True
 
     # Stash the dryrun value (if any) into the global configuration
-    sh.set_dryrun(args.pop('dryrun'))
+    sh.set_dryrun(args.get('dryrun', False))
 
+    # Params for the runner...
     dist = distro.Distro.get_current()
+    persona_inst = load_verify_persona(persona_fn, dist)
     config = cfg.get_config()
-    runner = actions.ActionRunner(dist, action, config, **args)
+    pw_gen = passwords.PasswordGenerator(config, args.get('prompt_for_passwords', True))
+    pkg_cls = dist.get_packager_factory()
+    pkg_manager = pkg_cls(dist, args.get('keep_old', False))
 
-    LOG.info("Starting action [%s] on %s for distro [%s]" % (action, date.rcf8222date(), dist))
+    runner = actions.ActionRunner(dist, action,
+                                  config, pw_gen,
+                                  pkg_manager,
+                                  **args)
+
+    LOG.info("Starting action [%s] on %s for distro: %s" % (action, date.rcf8222date(), dist.name))
+    LOG.info("Using persona: %s" % (persona_inst))
+    LOG.info("In root directory: %s" % (root_dir))
 
     start_time = time.time()
-    runner.run(persona_fn, root_dir)
+    runner.run(persona_inst, root_dir)
     end_time = time.time()
 
     LOG.info("It took (%s) to complete action [%s]" %