x/anvil
Code Issues Proposed changes

Added new audit level which is used for shell commands, good for dry run mode

Browse Source
This commit is contained in:
Joshua Harlow 2012-03-12 17:41:14 -07:00
parent 2ac8cd7aee
commit 9f7a32148d
8 changed files with 88 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
devstack/cfg.py
View File

@ -26,7 +26,6 @@ from devstack import utils
LOG = logging.getLogger("devstack.cfg") LOG = logging.getLogger("devstack.cfg")
PW_TMPL = "Enter a password for %s: "
ENV_PAT = re.compile(r"^\s*\$\{([\w\d]+):\-(.*)\}\s*$") ENV_PAT = re.compile(r"^\s*\$\{([\w\d]+):\-(.*)\}\s*$")
SUB_MATCH = re.compile(r"(?:\$\(([\w\d]+):([\w\d]+))\)") SUB_MATCH = re.compile(r"(?:\$\(([\w\d]+):([\w\d]+))\)")
CACHE_MSG = "(value will now be internally cached)" CACHE_MSG = "(value will now be internally cached)"

5
devstack/colorlog.py
View File

@ -29,6 +29,8 @@ COLOR_ATTRS = {
logging.CRITICAL: ['bold', 'blink'], logging.CRITICAL: ['bold', 'blink'],
} }
UNKNOWN_COLOR = 'grey'
class TermFormatter(logging.Formatter): class TermFormatter(logging.Formatter):
def __init__(self, reg_fmt=None, date_format=None): def __init__(self, reg_fmt=None, date_format=None):
@ -36,8 +38,7 @@ class TermFormatter(logging.Formatter):
def format(self, record): def format(self, record):
lvl = record.levelno lvl = record.levelno
color = COLOR_MAP.get(lvl) color = COLOR_MAP.get(lvl, UNKNOWN_COLOR)
if color:
record.levelname = colored(record.levelname, color) record.levelname = colored(record.levelname, color)
attrs = COLOR_ATTRS.get(lvl) attrs = COLOR_ATTRS.get(lvl)
if attrs: if attrs:

1
devstack/components/db.py
View File

@ -85,6 +85,7 @@ REQ_PKGS = ['db.json']
#config keys we warm up so u won't be prompted later #config keys we warm up so u won't be prompted later
WARMUP_PWS = ['sql'] WARMUP_PWS = ['sql']
#partial of database user prompt
PASSWORD_DESCRIPTION = 'the database user' PASSWORD_DESCRIPTION = 'the database user'

7
devstack/components/rabbit.py
View File

@ -44,6 +44,9 @@ WAIT_ON_TIME = settings.WAIT_ALIVE_SECS
#config keys we warm up so u won't be prompted later #config keys we warm up so u won't be prompted later
WARMUP_PWS = ['rabbit'] WARMUP_PWS = ['rabbit']
#partial of rabbit user prompt
PW_USER_PROMPT = 'the rabbit user'
class RabbitUninstaller(comp.PkgUninstallComponent): class RabbitUninstaller(comp.PkgUninstallComponent):
def __init__(self, *args, **kargs): def __init__(self, *args, **kargs):
@ -68,12 +71,12 @@ class RabbitInstaller(comp.PkgInstallComponent):
def warm_configs(self): def warm_configs(self):
for pw_key in WARMUP_PWS: for pw_key in WARMUP_PWS:
self.password_generator.get_password("passwords", pw_key, 'the rabbit user') self.password_generator.get_password("passwords", pw_key, PW_USER_PROMPT)
def _setup_pw(self): def _setup_pw(self):
LOG.info("Setting up your rabbit-mq guest password.") LOG.info("Setting up your rabbit-mq guest password.")
self.runtime.restart() self.runtime.restart()
passwd = self.password_generator.get_password('passwords', "rabbit", 'the rabbit user') passwd = self.password_generator.get_password('passwords', "rabbit", PW_USER_PROMPT)
cmd = PWD_CMD + [passwd] cmd = PWD_CMD + [passwd]
sh.execute(*cmd, run_as_root=True) sh.execute(*cmd, run_as_root=True)
LOG.info("Restarting so that your rabbit-mq guest password is reflected.") LOG.info("Restarting so that your rabbit-mq guest password is reflected.")

26
devstack/log.py
View File

@ -24,7 +24,7 @@ import pprint
from logging.handlers import SysLogHandler from logging.handlers import SysLogHandler
from logging.handlers import WatchedFileHandler from logging.handlers import WatchedFileHandler
# A list of things we want to replicate from logging levels # a list of things we want to replicate from logging levels
CRITICAL = logging.CRITICAL CRITICAL = logging.CRITICAL
FATAL = logging.FATAL FATAL = logging.FATAL
ERROR = logging.ERROR ERROR = logging.ERROR
@ -34,8 +34,13 @@ INFO = logging.INFO
DEBUG = logging.DEBUG DEBUG = logging.DEBUG
NOTSET = logging.NOTSET NOTSET = logging.NOTSET
# our new audit level
# http://docs.python.org/howto/logging.html#logging-levels
logging.AUDIT = logging.DEBUG + 1
logging.addLevelName(logging.AUDIT, 'AUDIT')
AUDIT = logging.AUDIT
# methods # methods
getLogger = logging.getLogger
debug = logging.debug debug = logging.debug
info = logging.info info = logging.info
warning = logging.warning warning = logging.warning
@ -55,6 +60,23 @@ WatchedFileHandler = WatchedFileHandler
SysLogHandler = SysLogHandler SysLogHandler = SysLogHandler
class AuditAdapter(logging.LoggerAdapter):
warn = logging.LoggerAdapter.warning
def __init__(self, logger):
self.logger = logger
def audit(self, msg, *args, **kwargs):
self.log(logging.AUDIT, msg, *args, **kwargs)
def process(self, msg, kwargs):
return msg, kwargs
def getLogger(name='devstack'):
return AuditAdapter(logging.getLogger(name))
def log_debug(f): def log_debug(f):
@functools.wraps(f) @functools.wraps(f)
def wrapper(*args, **kw): def wrapper(*args, **kw):

6
devstack/passwords.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python #!/usr/bin/env python
import binascii
import ConfigParser import ConfigParser
import binascii
import getpass import getpass
import logging import logging
import os import os
@ -53,7 +53,7 @@ class PasswordGenerator(object):
# FIXME: Remove the "section" argument, since it is always the same. # FIXME: Remove the "section" argument, since it is always the same.
def get_password(self, section, option, prompt_text, length=8): def get_password(self, section, option, prompt_text, length=8):
"""Returns a password identified by the configuration location.""" """Returns a password identified by the configuration location."""
LOG.debug('looking for password %s (%s)', option, prompt_text) LOG.debug('Looking for password %s (%s)', option, prompt_text)
# Look in the configuration file(s) # Look in the configuration file(s)
try: try:
@ -68,7 +68,7 @@ class PasswordGenerator(object):
# If we still don't have a value, make one up. # If we still don't have a value, make one up.
if not password: if not password:
LOG.debug('no configured password for %s (%s)', LOG.debug('No configured password for %s (%s)',
option, prompt_text) option, prompt_text)
password = generate_random(length) password = generate_random(length)

70
devstack/shell.py
View File

@ -101,15 +101,15 @@ def execute(*cmd, **kwargs):
execute_cmd = str_cmd.strip() execute_cmd = str_cmd.strip()
if not shell: if not shell:
LOG.debug('Running cmd: %s' % (execute_cmd)) LOG.audit('Running cmd: %s' % (execute_cmd))
else: else:
LOG.debug('Running shell cmd: %s' % (execute_cmd)) LOG.audit('Running shell cmd: %s' % (execute_cmd))
if process_input is not None: if process_input is not None:
LOG.debug('With stdin: %s' % (process_input)) LOG.audit('With stdin: %s' % (process_input))
if cwd: if cwd:
LOG.debug("In working directory: %s" % (cwd)) LOG.audit("In working directory: %s" % (cwd))
stdin_fh = subprocess.PIPE stdin_fh = subprocess.PIPE
stdout_fh = subprocess.PIPE stdout_fh = subprocess.PIPE
@ -132,7 +132,7 @@ def execute(*cmd, **kwargs):
process_env = None process_env = None
if env_overrides and len(env_overrides): if env_overrides and len(env_overrides):
process_env = env.get() process_env = env.get()
LOG.debug("With additional environment overrides: %s" % (env_overrides)) LOG.audit("With additional environment overrides: %s" % (env_overrides))
for (k, v) in env_overrides.items(): for (k, v) in env_overrides.items():
process_env[k] = str(v) process_env[k] = str(v)
@ -140,7 +140,6 @@ def execute(*cmd, **kwargs):
result = None result = None
with Rooted(run_as_root): with Rooted(run_as_root):
if DRYRUN_MODE: if DRYRUN_MODE:
LOG.debug("Fake executing: %s" % (execute_cmd))
rc = DRY_RC rc = DRY_RC
result = DRY_STDOUT_ERR result = DRY_STDOUT_ERR
else: else:
@ -164,7 +163,7 @@ def execute(*cmd, **kwargs):
and obj.stdin and close_stdin): and obj.stdin and close_stdin):
obj.stdin.close() obj.stdin.close()
rc = obj.returncode rc = obj.returncode
LOG.debug('Cmd result had exit code: %s' % rc) LOG.audit('Cmd result had exit code: %s' % rc)
if not result: if not result:
result = ("", "") result = ("", "")
@ -241,21 +240,19 @@ def _get_suids():
return (uid, gid) return (uid, gid)
def chown_r(path, uid, gid, run_as_root=True): def chown_r(path, uid, gid, run_as_root=True):
with Rooted(run_as_root): with Rooted(run_as_root):
if isdir(path): if isdir(path):
LOG.debug("Changing ownership of %s to %s:%s" % (path, uid, gid)) LOG.audit("Changing ownership of %s to %s:%s" % (path, uid, gid))
for root, dirs, files in os.walk(path): for root, dirs, files in os.walk(path):
os.chown(root, uid, gid) os.chown(root, uid, gid)
LOG.debug("Changing ownership of %s to %s:%s" % (root, uid, gid)) LOG.debug("Changing ownership of %s to %s:%s" % (root, uid, gid))
for d in dirs: for d in dirs:
os.chown(joinpths(root, d), uid, gid) os.chown(joinpths(root, d), uid, gid)
LOG.debug("Changing ownership of %s to %s:%s" % (joinpths(root, d), uid, gid)) LOG.audit("Changing ownership of %s to %s:%s" % (joinpths(root, d), uid, gid))
for f in files: for f in files:
os.chown(joinpths(root, f), uid, gid) os.chown(joinpths(root, f), uid, gid)
LOG.debug("Changing ownership of %s to %s:%s" % (joinpths(root, f), uid, gid)) LOG.audit("Changing ownership of %s to %s:%s" % (joinpths(root, f), uid, gid))
@ -327,8 +324,8 @@ def mkdirslist(path):
def append_file(fn, text, flush=True, quiet=False): def append_file(fn, text, flush=True, quiet=False):
if not quiet: if not quiet:
LOG.debug("Appending to file %s (%d bytes) (%s)", fn, len(text), flush) LOG.audit("Appending to file %s (%d bytes) (%s)", fn, len(text), flush)
LOG.debug(">> %s" % (text)) LOG.audit(">> %s" % (text))
if not DRYRUN_MODE: if not DRYRUN_MODE:
with open(fn, "a") as f: with open(fn, "a") as f:
f.write(text) f.write(text)
@ -339,8 +336,8 @@ def append_file(fn, text, flush=True, quiet=False):
def write_file(fn, text, flush=True, quiet=False): def write_file(fn, text, flush=True, quiet=False):
if not quiet: if not quiet:
LOG.debug("Writing to file %s (%d bytes)", fn, len(text)) LOG.audit("Writing to file %s (%d bytes)", fn, len(text))
LOG.debug("> %s" % (text)) LOG.audit("> %s" % (text))
if not DRYRUN_MODE: if not DRYRUN_MODE:
with open(fn, "w") as f: with open(fn, "w") as f:
f.write(text) f.write(text)
@ -352,7 +349,7 @@ def write_file(fn, text, flush=True, quiet=False):
def touch_file(fn, die_if_there=True, quiet=False, file_size=0): def touch_file(fn, die_if_there=True, quiet=False, file_size=0):
if not isfile(fn): if not isfile(fn):
if not quiet: if not quiet:
LOG.debug("Touching and truncating file %s (%s)", fn, file_size) LOG.audit("Touching and truncating file %s (%s)", fn, file_size)
if not DRYRUN_MODE: if not DRYRUN_MODE:
with open(fn, "w") as f: with open(fn, "w") as f:
f.truncate(file_size) f.truncate(file_size)
@ -365,29 +362,31 @@ def touch_file(fn, die_if_there=True, quiet=False, file_size=0):
def load_file(fn, quiet=False): def load_file(fn, quiet=False):
if not quiet: if not quiet:
LOG.debug("Loading data from file %s", fn) LOG.audit("Loading data from file %s", fn)
data = "" data = ""
try: try:
with open(fn, "r") as f: with open(fn, "r") as f:
data = f.read() data = f.read()
except IOError as e: except IOError as e:
if DRYRUN_MODE: if DRYRUN_MODE:
LOG.debug("Passing on load exception since in dry-run mode") # We still need to actually load something (ie the json install files so thats)
# Why this is in the exception path.
LOG.audit("Passing on load exception since in dry-run mode")
else: else:
raise e raise e
if not quiet: if not quiet:
LOG.debug("Loaded (%d) bytes from file %s", len(data), fn) LOG.audit("Loaded (%d) bytes from file %s", len(data), fn)
return data return data
def mkdir(path, recurse=True): def mkdir(path, recurse=True):
if not isdir(path): if not isdir(path):
if recurse: if recurse:
LOG.debug("Recursively creating directory \"%s\"" % (path)) LOG.audit("Recursively creating directory \"%s\"" % (path))
if not DRYRUN_MODE: if not DRYRUN_MODE:
os.makedirs(path) os.makedirs(path)
else: else:
LOG.debug("Creating directory \"%s\"" % (path)) LOG.audit("Creating directory \"%s\"" % (path))
if not DRYRUN_MODE: if not DRYRUN_MODE:
os.mkdir(path) os.mkdir(path)
@ -395,7 +394,7 @@ def mkdir(path, recurse=True):
def deldir(path, run_as_root=False): def deldir(path, run_as_root=False):
with Rooted(run_as_root): with Rooted(run_as_root):
if isdir(path): if isdir(path):
LOG.debug("Recursively deleting directory tree starting at \"%s\"" % (path)) LOG.audit("Recursively deleting directory tree starting at \"%s\"" % (path))
if not DRYRUN_MODE: if not DRYRUN_MODE:
shutil.rmtree(path) shutil.rmtree(path)
@ -404,11 +403,11 @@ def rmdir(path, quiet=True, run_as_root=False):
if not isdir(path): if not isdir(path):
return return
try: try:
with Rooted(run_as_root): with audit(run_as_root):
LOG.debug("Deleting directory \"%s\" with the cavet that we will fail if it's not empty." % (path)) LOG.debug("Deleting directory \"%s\" with the cavet that we will fail if it's not empty." % (path))
if not DRYRUN_MODE: if not DRYRUN_MODE:
os.rmdir(path) os.rmdir(path)
LOG.debug("Deleted directory \"%s\"" % (path)) LOG.audit("Deleted directory \"%s\"" % (path))
except OSError: except OSError:
if not quiet: if not quiet:
raise raise
@ -418,7 +417,7 @@ def rmdir(path, quiet=True, run_as_root=False):
def symlink(source, link, force=True, run_as_root=True): def symlink(source, link, force=True, run_as_root=True):
with Rooted(run_as_root): with Rooted(run_as_root):
LOG.debug("Creating symlink from %s => %s" % (link, source)) LOG.audit("Creating symlink from %s => %s" % (link, source))
path = dirname(link) path = dirname(link)
needed_pths = mkdirslist(path) needed_pths = mkdirslist(path)
if not DRYRUN_MODE: if not DRYRUN_MODE:
@ -534,9 +533,9 @@ def umount(dev_name, ignore_errors=True):
def unlink(path, ignore_errors=True, run_as_root=False): def unlink(path, ignore_errors=True, run_as_root=False):
try: LOG.audit("Unlinking (removing) %s" % (path))
LOG.debug("Unlinking (removing) %s" % (path))
if not DRYRUN_MODE: if not DRYRUN_MODE:
try:
with Rooted(run_as_root): with Rooted(run_as_root):
os.unlink(path) os.unlink(path)
except OSError: except OSError:
@ -547,17 +546,15 @@ def unlink(path, ignore_errors=True, run_as_root=False):
def move(src, dst): def move(src, dst):
if DRYRUN_MODE: LOG.audit("Moving: %s => %s" % (src, dst))
LOG.debug("Faking move from: %s => %s" % (src, dst)) if not DRYRUN_MODE:
else:
shutil.move(src, dst) shutil.move(src, dst)
return dst return dst
def chmod(fname, mode): def chmod(fname, mode):
if DRYRUN_MODE: LOG.audit("Applying chmod: %s to %s" % (fname, mode))
LOG.debug("Faking chmod: %s to %s" % (fname, mode)) if not DRYRUN_MODE:
else:
os.chmod(fname, mode) os.chmod(fname, mode)
return fname return fname
@ -576,9 +573,8 @@ def replace_in(fn, search, replace, run_as_root=False):
def copy_replace_file(fsrc, fdst, linemap): def copy_replace_file(fsrc, fdst, linemap):
files = mkdirslist(dirname(fdst)) files = mkdirslist(dirname(fdst))
if DRYRUN_MODE:
LOG.debug("Copying and replacing file: %s => %s" % (fsrc, fdst)) LOG.debug("Copying and replacing file: %s => %s" % (fsrc, fdst))
else: if not DRYRUN_MODE:
with open(fdst, 'w') as fh: with open(fdst, 'w') as fh:
for line in fileinput.input(fsrc): for line in fileinput.input(fsrc):
for (k, v) in linemap.items(): for (k, v) in linemap.items():
@ -642,6 +638,6 @@ def getegid():
def sleep(winks): def sleep(winks):
if DRYRUN_MODE: if DRYRUN_MODE:
LOG.debug("Not really sleeping for: %s seconds" % (winks)) LOG.audit("Not really sleeping for: %s seconds" % (winks))
else: else:
time.sleep(winks) time.sleep(winks)

20
stack
View File

@ -16,15 +16,13 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import logging
import logging.config
import sys import sys
import time import time
import traceback import traceback
from devstack import colorlog from devstack import colorlog
from devstack import date from devstack import date
from devstack import log as lg from devstack import log as logging
from devstack import opts from devstack import opts
from devstack import passwords from devstack import passwords
from devstack import settings from devstack import settings
@ -34,7 +32,7 @@ from devstack import utils
from devstack.progs import actions from devstack.progs import actions
from devstack.progs import common from devstack.progs import common
LOG = lg.getLogger("devstack.stack") LOG = logging.getLogger("devstack.stack")
# This is used to map an action to a useful string for # This is used to map an action to a useful string for
# the welcome display # the welcome display
@ -96,9 +94,9 @@ def run(args):
config = common.get_config() config = common.get_config()
# Stash the dryrun value (if any) into the global configuration # Stash the dryrun value (if any) into the global configuration
sh.set_dryrun(args.pop('dryrun')) sh.set_dryrun(args['dryrun'])
password_generator = passwords.PasswordGenerator(config, args['prompt_for_passwords']) password_generator = passwords.PasswordGenerator(config, args['prompt_for_passwords'])
pkg_manager = common.get_packager(distro, args.get('keep_old')) pkg_manager = common.get_packager(distro, args['keep_old'])
components = utils.parse_components(args.pop("components")) components = utils.parse_components(args.pop("components"))
runner = actions.ActionRunner(distro, action, rootdir, config, password_generator, runner = actions.ActionRunner(distro, action, rootdir, config, password_generator,
pkg_manager, components=components, **args) pkg_manager, components=components, **args)
@ -113,11 +111,11 @@ def run(args):
def configure_logging(args): def configure_logging(args):
# Debug by default # Debug by default
root_logger = logging.getLogger('') root_logger = logging.getLogger().logger
root_logger.setLevel(logging.DEBUG) root_logger.setLevel(logging.DEBUG)
# Set our pretty logger # Set our pretty logger
console_logger = logging.StreamHandler(sys.stdout) console_logger = logging.StreamHandler(stream=sys.stdout)
console_format = '%(levelname)s: @%(name)s : %(message)s' console_format = '%(levelname)s: @%(name)s : %(message)s'
if sh.in_terminal(): if sh.in_terminal():
console_logger.setFormatter(colorlog.TermFormatter(console_format)) console_logger.setFormatter(colorlog.TermFormatter(console_format))
@ -126,7 +124,11 @@ def configure_logging(args):
root_logger.addHandler(console_logger) root_logger.addHandler(console_logger)
# Adjust logging verbose level based on the command line switch. # Adjust logging verbose level based on the command line switch.
log_level = logging.DEBUG if args['verbosity'] >= 2 else logging.INFO log_level = logging.INFO
if args['verbosity'] >= 2:
log_level = logging.DEBUG
elif args['dryrun']:
log_level = logging.AUDIT
root_logger.setLevel(log_level) root_logger.setLevel(log_level)