From 9f7a32148dfd5831febdf89c3c22fbaa7cb318be Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 12 Mar 2012 17:41:14 -0700 Subject: [PATCH] Added new audit level which is used for shell commands, good for dry run mode --- devstack/cfg.py | 1 - devstack/colorlog.py | 7 +-- devstack/components/db.py | 1 + devstack/components/rabbit.py | 7 ++- devstack/log.py | 26 ++++++++++- devstack/passwords.py | 6 +-- devstack/shell.py | 84 +++++++++++++++++------------------ stack | 20 +++++---- 8 files changed, 88 insertions(+), 64 deletions(-) diff --git a/devstack/cfg.py b/devstack/cfg.py index 46c46ec6..0aa489c5 100644 --- a/devstack/cfg.py +++ b/devstack/cfg.py @@ -26,7 +26,6 @@ from devstack import utils LOG = logging.getLogger("devstack.cfg") -PW_TMPL = "Enter a password for %s: " ENV_PAT = re.compile(r"^\s*\$\{([\w\d]+):\-(.*)\}\s*$") SUB_MATCH = re.compile(r"(?:\$\(([\w\d]+):([\w\d]+))\)") CACHE_MSG = "(value will now be internally cached)" diff --git a/devstack/colorlog.py b/devstack/colorlog.py index 88e6befc..e68f99bf 100644 --- a/devstack/colorlog.py +++ b/devstack/colorlog.py @@ -29,6 +29,8 @@ COLOR_ATTRS = { logging.CRITICAL: ['bold', 'blink'], } +UNKNOWN_COLOR = 'grey' + class TermFormatter(logging.Formatter): def __init__(self, reg_fmt=None, date_format=None): @@ -36,9 +38,8 @@ class TermFormatter(logging.Formatter): def format(self, record): lvl = record.levelno - color = COLOR_MAP.get(lvl) - if color: - record.levelname = colored(record.levelname, color) + color = COLOR_MAP.get(lvl, UNKNOWN_COLOR) + record.levelname = colored(record.levelname, color) attrs = COLOR_ATTRS.get(lvl) if attrs: record.msg = colored(record.msg, attrs=attrs) diff --git a/devstack/components/db.py b/devstack/components/db.py index f00c54ac..5e9c2de8 100644 --- a/devstack/components/db.py +++ b/devstack/components/db.py @@ -85,6 +85,7 @@ REQ_PKGS = ['db.json'] #config keys we warm up so u won't be prompted later WARMUP_PWS = ['sql'] +#partial of database user prompt PASSWORD_DESCRIPTION = 'the database user' diff --git a/devstack/components/rabbit.py b/devstack/components/rabbit.py index 6eb89f88..6eac0d79 100644 --- a/devstack/components/rabbit.py +++ b/devstack/components/rabbit.py @@ -44,6 +44,9 @@ WAIT_ON_TIME = settings.WAIT_ALIVE_SECS #config keys we warm up so u won't be prompted later WARMUP_PWS = ['rabbit'] +#partial of rabbit user prompt +PW_USER_PROMPT = 'the rabbit user' + class RabbitUninstaller(comp.PkgUninstallComponent): def __init__(self, *args, **kargs): @@ -68,12 +71,12 @@ class RabbitInstaller(comp.PkgInstallComponent): def warm_configs(self): for pw_key in WARMUP_PWS: - self.password_generator.get_password("passwords", pw_key, 'the rabbit user') + self.password_generator.get_password("passwords", pw_key, PW_USER_PROMPT) def _setup_pw(self): LOG.info("Setting up your rabbit-mq guest password.") self.runtime.restart() - passwd = self.password_generator.get_password('passwords', "rabbit", 'the rabbit user') + passwd = self.password_generator.get_password('passwords', "rabbit", PW_USER_PROMPT) cmd = PWD_CMD + [passwd] sh.execute(*cmd, run_as_root=True) LOG.info("Restarting so that your rabbit-mq guest password is reflected.") diff --git a/devstack/log.py b/devstack/log.py index 0d989532..8e140c8e 100644 --- a/devstack/log.py +++ b/devstack/log.py @@ -24,7 +24,7 @@ import pprint from logging.handlers import SysLogHandler from logging.handlers import WatchedFileHandler -# A list of things we want to replicate from logging levels +# a list of things we want to replicate from logging levels CRITICAL = logging.CRITICAL FATAL = logging.FATAL ERROR = logging.ERROR @@ -34,8 +34,13 @@ INFO = logging.INFO DEBUG = logging.DEBUG NOTSET = logging.NOTSET +# our new audit level +# http://docs.python.org/howto/logging.html#logging-levels +logging.AUDIT = logging.DEBUG + 1 +logging.addLevelName(logging.AUDIT, 'AUDIT') +AUDIT = logging.AUDIT + # methods -getLogger = logging.getLogger debug = logging.debug info = logging.info warning = logging.warning @@ -55,6 +60,23 @@ WatchedFileHandler = WatchedFileHandler SysLogHandler = SysLogHandler +class AuditAdapter(logging.LoggerAdapter): + warn = logging.LoggerAdapter.warning + + def __init__(self, logger): + self.logger = logger + + def audit(self, msg, *args, **kwargs): + self.log(logging.AUDIT, msg, *args, **kwargs) + + def process(self, msg, kwargs): + return msg, kwargs + + +def getLogger(name='devstack'): + return AuditAdapter(logging.getLogger(name)) + + def log_debug(f): @functools.wraps(f) def wrapper(*args, **kw): diff --git a/devstack/passwords.py b/devstack/passwords.py index 47fdd70a..21b9774a 100644 --- a/devstack/passwords.py +++ b/devstack/passwords.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -import binascii import ConfigParser +import binascii import getpass import logging import os @@ -53,7 +53,7 @@ class PasswordGenerator(object): # FIXME: Remove the "section" argument, since it is always the same. def get_password(self, section, option, prompt_text, length=8): """Returns a password identified by the configuration location.""" - LOG.debug('looking for password %s (%s)', option, prompt_text) + LOG.debug('Looking for password %s (%s)', option, prompt_text) # Look in the configuration file(s) try: @@ -68,7 +68,7 @@ class PasswordGenerator(object): # If we still don't have a value, make one up. if not password: - LOG.debug('no configured password for %s (%s)', + LOG.debug('No configured password for %s (%s)', option, prompt_text) password = generate_random(length) diff --git a/devstack/shell.py b/devstack/shell.py index b5834293..7c028a79 100644 --- a/devstack/shell.py +++ b/devstack/shell.py @@ -101,15 +101,15 @@ def execute(*cmd, **kwargs): execute_cmd = str_cmd.strip() if not shell: - LOG.debug('Running cmd: %s' % (execute_cmd)) + LOG.audit('Running cmd: %s' % (execute_cmd)) else: - LOG.debug('Running shell cmd: %s' % (execute_cmd)) + LOG.audit('Running shell cmd: %s' % (execute_cmd)) if process_input is not None: - LOG.debug('With stdin: %s' % (process_input)) + LOG.audit('With stdin: %s' % (process_input)) if cwd: - LOG.debug("In working directory: %s" % (cwd)) + LOG.audit("In working directory: %s" % (cwd)) stdin_fh = subprocess.PIPE stdout_fh = subprocess.PIPE @@ -132,7 +132,7 @@ def execute(*cmd, **kwargs): process_env = None if env_overrides and len(env_overrides): process_env = env.get() - LOG.debug("With additional environment overrides: %s" % (env_overrides)) + LOG.audit("With additional environment overrides: %s" % (env_overrides)) for (k, v) in env_overrides.items(): process_env[k] = str(v) @@ -140,7 +140,6 @@ def execute(*cmd, **kwargs): result = None with Rooted(run_as_root): if DRYRUN_MODE: - LOG.debug("Fake executing: %s" % (execute_cmd)) rc = DRY_RC result = DRY_STDOUT_ERR else: @@ -164,7 +163,7 @@ def execute(*cmd, **kwargs): and obj.stdin and close_stdin): obj.stdin.close() rc = obj.returncode - LOG.debug('Cmd result had exit code: %s' % rc) + LOG.audit('Cmd result had exit code: %s' % rc) if not result: result = ("", "") @@ -241,21 +240,19 @@ def _get_suids(): return (uid, gid) - - def chown_r(path, uid, gid, run_as_root=True): with Rooted(run_as_root): if isdir(path): - LOG.debug("Changing ownership of %s to %s:%s" % (path, uid, gid)) + LOG.audit("Changing ownership of %s to %s:%s" % (path, uid, gid)) for root, dirs, files in os.walk(path): os.chown(root, uid, gid) LOG.debug("Changing ownership of %s to %s:%s" % (root, uid, gid)) for d in dirs: os.chown(joinpths(root, d), uid, gid) - LOG.debug("Changing ownership of %s to %s:%s" % (joinpths(root, d), uid, gid)) + LOG.audit("Changing ownership of %s to %s:%s" % (joinpths(root, d), uid, gid)) for f in files: os.chown(joinpths(root, f), uid, gid) - LOG.debug("Changing ownership of %s to %s:%s" % (joinpths(root, f), uid, gid)) + LOG.audit("Changing ownership of %s to %s:%s" % (joinpths(root, f), uid, gid)) @@ -327,8 +324,8 @@ def mkdirslist(path): def append_file(fn, text, flush=True, quiet=False): if not quiet: - LOG.debug("Appending to file %s (%d bytes) (%s)", fn, len(text), flush) - LOG.debug(">> %s" % (text)) + LOG.audit("Appending to file %s (%d bytes) (%s)", fn, len(text), flush) + LOG.audit(">> %s" % (text)) if not DRYRUN_MODE: with open(fn, "a") as f: f.write(text) @@ -339,8 +336,8 @@ def append_file(fn, text, flush=True, quiet=False): def write_file(fn, text, flush=True, quiet=False): if not quiet: - LOG.debug("Writing to file %s (%d bytes)", fn, len(text)) - LOG.debug("> %s" % (text)) + LOG.audit("Writing to file %s (%d bytes)", fn, len(text)) + LOG.audit("> %s" % (text)) if not DRYRUN_MODE: with open(fn, "w") as f: f.write(text) @@ -352,7 +349,7 @@ def write_file(fn, text, flush=True, quiet=False): def touch_file(fn, die_if_there=True, quiet=False, file_size=0): if not isfile(fn): if not quiet: - LOG.debug("Touching and truncating file %s (%s)", fn, file_size) + LOG.audit("Touching and truncating file %s (%s)", fn, file_size) if not DRYRUN_MODE: with open(fn, "w") as f: f.truncate(file_size) @@ -365,29 +362,31 @@ def touch_file(fn, die_if_there=True, quiet=False, file_size=0): def load_file(fn, quiet=False): if not quiet: - LOG.debug("Loading data from file %s", fn) + LOG.audit("Loading data from file %s", fn) data = "" try: with open(fn, "r") as f: data = f.read() except IOError as e: if DRYRUN_MODE: - LOG.debug("Passing on load exception since in dry-run mode") + # We still need to actually load something (ie the json install files so thats) + # Why this is in the exception path. + LOG.audit("Passing on load exception since in dry-run mode") else: raise e if not quiet: - LOG.debug("Loaded (%d) bytes from file %s", len(data), fn) + LOG.audit("Loaded (%d) bytes from file %s", len(data), fn) return data def mkdir(path, recurse=True): if not isdir(path): if recurse: - LOG.debug("Recursively creating directory \"%s\"" % (path)) + LOG.audit("Recursively creating directory \"%s\"" % (path)) if not DRYRUN_MODE: os.makedirs(path) else: - LOG.debug("Creating directory \"%s\"" % (path)) + LOG.audit("Creating directory \"%s\"" % (path)) if not DRYRUN_MODE: os.mkdir(path) @@ -395,7 +394,7 @@ def mkdir(path, recurse=True): def deldir(path, run_as_root=False): with Rooted(run_as_root): if isdir(path): - LOG.debug("Recursively deleting directory tree starting at \"%s\"" % (path)) + LOG.audit("Recursively deleting directory tree starting at \"%s\"" % (path)) if not DRYRUN_MODE: shutil.rmtree(path) @@ -404,11 +403,11 @@ def rmdir(path, quiet=True, run_as_root=False): if not isdir(path): return try: - with Rooted(run_as_root): + with audit(run_as_root): LOG.debug("Deleting directory \"%s\" with the cavet that we will fail if it's not empty." % (path)) if not DRYRUN_MODE: os.rmdir(path) - LOG.debug("Deleted directory \"%s\"" % (path)) + LOG.audit("Deleted directory \"%s\"" % (path)) except OSError: if not quiet: raise @@ -418,7 +417,7 @@ def rmdir(path, quiet=True, run_as_root=False): def symlink(source, link, force=True, run_as_root=True): with Rooted(run_as_root): - LOG.debug("Creating symlink from %s => %s" % (link, source)) + LOG.audit("Creating symlink from %s => %s" % (link, source)) path = dirname(link) needed_pths = mkdirslist(path) if not DRYRUN_MODE: @@ -534,30 +533,28 @@ def umount(dev_name, ignore_errors=True): def unlink(path, ignore_errors=True, run_as_root=False): - try: - LOG.debug("Unlinking (removing) %s" % (path)) - if not DRYRUN_MODE: + LOG.audit("Unlinking (removing) %s" % (path)) + if not DRYRUN_MODE: + try: with Rooted(run_as_root): os.unlink(path) - except OSError: - if not ignore_errors: - raise - else: - pass + except OSError: + if not ignore_errors: + raise + else: + pass def move(src, dst): - if DRYRUN_MODE: - LOG.debug("Faking move from: %s => %s" % (src, dst)) - else: + LOG.audit("Moving: %s => %s" % (src, dst)) + if not DRYRUN_MODE: shutil.move(src, dst) return dst def chmod(fname, mode): - if DRYRUN_MODE: - LOG.debug("Faking chmod: %s to %s" % (fname, mode)) - else: + LOG.audit("Applying chmod: %s to %s" % (fname, mode)) + if not DRYRUN_MODE: os.chmod(fname, mode) return fname @@ -576,9 +573,8 @@ def replace_in(fn, search, replace, run_as_root=False): def copy_replace_file(fsrc, fdst, linemap): files = mkdirslist(dirname(fdst)) - if DRYRUN_MODE: - LOG.debug("Copying and replacing file: %s => %s" % (fsrc, fdst)) - else: + LOG.debug("Copying and replacing file: %s => %s" % (fsrc, fdst)) + if not DRYRUN_MODE: with open(fdst, 'w') as fh: for line in fileinput.input(fsrc): for (k, v) in linemap.items(): @@ -642,6 +638,6 @@ def getegid(): def sleep(winks): if DRYRUN_MODE: - LOG.debug("Not really sleeping for: %s seconds" % (winks)) + LOG.audit("Not really sleeping for: %s seconds" % (winks)) else: time.sleep(winks) diff --git a/stack b/stack index 809c9b5f..c474f790 100755 --- a/stack +++ b/stack @@ -16,15 +16,13 @@ # License for the specific language governing permissions and limitations # under the License. -import logging -import logging.config import sys import time import traceback from devstack import colorlog from devstack import date -from devstack import log as lg +from devstack import log as logging from devstack import opts from devstack import passwords from devstack import settings @@ -34,7 +32,7 @@ from devstack import utils from devstack.progs import actions from devstack.progs import common -LOG = lg.getLogger("devstack.stack") +LOG = logging.getLogger("devstack.stack") # This is used to map an action to a useful string for # the welcome display @@ -96,9 +94,9 @@ def run(args): config = common.get_config() # Stash the dryrun value (if any) into the global configuration - sh.set_dryrun(args.pop('dryrun')) + sh.set_dryrun(args['dryrun']) password_generator = passwords.PasswordGenerator(config, args['prompt_for_passwords']) - pkg_manager = common.get_packager(distro, args.get('keep_old')) + pkg_manager = common.get_packager(distro, args['keep_old']) components = utils.parse_components(args.pop("components")) runner = actions.ActionRunner(distro, action, rootdir, config, password_generator, pkg_manager, components=components, **args) @@ -113,11 +111,11 @@ def run(args): def configure_logging(args): # Debug by default - root_logger = logging.getLogger('') + root_logger = logging.getLogger().logger root_logger.setLevel(logging.DEBUG) # Set our pretty logger - console_logger = logging.StreamHandler(sys.stdout) + console_logger = logging.StreamHandler(stream=sys.stdout) console_format = '%(levelname)s: @%(name)s : %(message)s' if sh.in_terminal(): console_logger.setFormatter(colorlog.TermFormatter(console_format)) @@ -126,7 +124,11 @@ def configure_logging(args): root_logger.addHandler(console_logger) # Adjust logging verbose level based on the command line switch. - log_level = logging.DEBUG if args['verbosity'] >= 2 else logging.INFO + log_level = logging.INFO + if args['verbosity'] >= 2: + log_level = logging.DEBUG + elif args['dryrun']: + log_level = logging.AUDIT root_logger.setLevel(log_level)