x/anvil
Code Issues Proposed changes

Cleanup aisle 10

Browse Source
This commit is contained in:
Joshua Harlow 2012-03-08 18:17:05 -08:00
parent e59151182c
commit cd17a83a60
12 changed files with 215 additions and 273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
devstack/component.py
View File

@ -77,16 +77,16 @@ class ComponentBase(object):
pass
def is_started(self):
return tr.TraceReader(self.tracedir, tr.START_TRACE).exists()
return tr.TraceReader(tr.trace_fn(self.tracedir, tr.START_TRACE)).exists()
def is_installed(self):
return tr.TraceReader(self.tracedir, tr.IN_TRACE).exists()
return tr.TraceReader(tr.trace_fn(self.tracedir, tr.IN_TRACE)).exists()
class PkgInstallComponent(ComponentBase):
def __init__(self, component_name, *args, **kargs):
ComponentBase.__init__(self, component_name, *args, **kargs)
self.tracewriter = tr.TraceWriter(self.tracedir, tr.IN_TRACE)
self.tracewriter = tr.TraceWriter(tr.trace_fn(self.tracedir, tr.IN_TRACE))
def _get_download_locations(self):
return list()
@ -109,7 +109,7 @@ class PkgInstallComponent(ComponentBase):
branch = self.cfg.get(cfg_section, cfg_key)
(cfg_section, cfg_key) = uri_tuple
uri = self.cfg.get(cfg_section, cfg_key)
self.tracewriter.downloaded(target_loc, uri)
self.tracewriter.download_happened(target_loc, uri)
dirs_made = down.download(target_loc, uri, branch)
#ensure this is always added so that
#if a keep old happens then this of course
@ -117,7 +117,7 @@ class PkgInstallComponent(ComponentBase):
#then this won't be deleted this time around
#adding it in is harmless and willl make sure its removed
dirs_made.append(target_loc)
self.tracewriter.dir_made(*dirs_made)
self.tracewriter.dirs_made(*dirs_made)
return len(locations)
def _get_param_map(self, config_fn):
@ -143,7 +143,7 @@ class PkgInstallComponent(ComponentBase):
LOG.info("Installing packages (%s)." % (", ".join(pkgnames)))
#do this before install just incase it craps out half way through
for name in pkgnames:
self.tracewriter.package_install(name, pkgs.get(name))
self.tracewriter.package_installed(name, pkgs.get(name))
#now actually install
self.packager.install_batch(pkgs)
return self.tracedir
@ -189,22 +189,20 @@ class PkgInstallComponent(ComponentBase):
for fn in configs:
#get the params and where it should come from and where it should go
parameters = self._get_param_map(fn)
tgtfn = self._get_target_config_name(fn)
tgt_fn = self._get_target_config_name(fn)
#ensure directory is there (if not created previously)
self.tracewriter.make_dir(sh.dirname(tgtfn))
self.tracewriter.dirs_made(*sh.mkdirslist(sh.dirname(tgt_fn)))
#now configure it
LOG.info("Configuring file %s" % (fn))
(sourcefn, contents) = self._get_source_config(fn)
LOG.debug("Replacing parameters in file %s" % (sourcefn))
(source_fn, contents) = self._get_source_config(fn)
LOG.debug("Replacing parameters in file %s" % (source_fn))
LOG.debug("Replacements = %s" % (parameters))
contents = utils.param_replace(contents, parameters)
LOG.debug("Applying side-effects of param replacement for template %s" % (sourcefn))
LOG.debug("Applying side-effects of param replacement for template %s" % (source_fn))
contents = self._config_adjust(contents, fn)
LOG.info("Writing configuration file %s" % (tgtfn))
LOG.info("Writing configuration file %s" % (tgt_fn))
#this trace is used to remove the files configured
#do this before write just incase it craps out half way through
self.tracewriter.cfg_write(tgtfn)
sh.write_file(tgtfn, contents)
self.tracewriter.cfg_file_written(sh.write_file(tgt_fn, contents))
return len(configs)
def _configure_symlinks(self):
@ -215,7 +213,8 @@ class PkgInstallComponent(ComponentBase):
link = links.get(source)
try:
LOG.info("Symlinking %s => %s" % (link, source))
self.tracewriter.symlink(source, link)
self.tracewriter.dirs_made(*sh.symlink(source, link))
self.tracewriter.symlink_made(link)
except OSError:
LOG.warn("Symlink %s => %s already exists." % (link, source))
return len(links)
@ -254,31 +253,28 @@ class PythonInstallComponent(PkgInstallComponent):
LOG.info("Setting up %s pips (%s)" % (len(pips), ", ".join(pips.keys())))
#do this before install just incase it craps out half way through
for name in pips.keys():
self.tracewriter.pip_install(name, pips.get(name))
self.tracewriter.pip_installed(name, pips.get(name))
#now install
pip.install(pips, self.distro)
def _format_stderr_out(self, stderr, stdout):
combined = ["===STDOUT===", str(stdout), "===STDERR===", str(stderr)]
return utils.joinlinesep(*combined)
def _format_trace_name(self, name):
return "%s-%s" % (tr.PY_TRACE, name)
def _install_python_setups(self):
pydirs = self._get_python_directories()
if pydirs:
LOG.info("Setting up %s python directories (%s)" % (len(pydirs), pydirs))
for (name, wkdir) in pydirs.items():
working_dir = wkdir or self.appdir
self.tracewriter.make_dir(working_dir)
record_fn = tr.touch_trace(self.tracedir, self._format_trace_name(name))
#ensure working dir is there
self.tracewriter.dirs_made(*sh.mkdirslist(working_dir))
#do this before write just incase it craps out half way through
self.tracewriter.file_touched(record_fn)
self.tracewriter.py_install(name, record_fn, working_dir)
self.tracewriter.py_installed(name, working_dir)
#now actually do it
(stdout, stderr) = sh.execute(*PY_INSTALL, cwd=working_dir, run_as_root=True)
sh.write_file(record_fn, self._format_stderr_out(stderr, stdout))
py_trace_name = "%s-%s" % (tr.PY_TRACE, name)
py_writer = tr.TraceWriter(tr.trace_fn(self.tracedir, py_trace_name))
py_writer.trace("CMD", " ".join(PY_INSTALL))
py_writer.trace("STDOUT", stdout)
py_writer.trace("STDERR", stderr)
self.tracewriter.file_touched(py_writer.filename())
def _python_install(self):
self._install_pips()
@ -293,7 +289,7 @@ class PythonInstallComponent(PkgInstallComponent):
class PkgUninstallComponent(ComponentBase):
def __init__(self, component_name, *args, **kargs):
ComponentBase.__init__(self, component_name, *args, **kargs)
self.tracereader = tr.TraceReader(self.tracedir, tr.IN_TRACE)
self.tracereader = tr.TraceReader(tr.trace_fn(self.tracedir, tr.IN_TRACE))
self.keep_old = kargs.get("keep_old")
def unconfigure(self):
@ -356,12 +352,8 @@ class PkgUninstallComponent(ComponentBase):
if dirsmade:
dirsmade = [sh.abspth(d) for d in dirsmade]
if self.keep_old:
downloads = (self.tracereader.downloaded())
places = set()
for info in downloads:
download_place = info.get('target')
if download_place:
places.add(download_place)
download_locations = self.tracereader.download_locations()
places = set(download_locations)
LOG.info("Keeping %s download directories [%s]" % (len(places), ",".join(sorted(places))))
for download_place in places:
dirsmade = sh.remove_parents(download_place, dirsmade)
@ -389,17 +381,15 @@ class PythonUninstallComponent(PkgUninstallComponent):
pylisting = self.tracereader.py_listing()
if pylisting:
LOG.info("Uninstalling %s python setups." % (len(pylisting)))
for entry in pylisting:
where = entry.get('where')
for (_, where) in pylisting:
sh.execute(*PY_UNINSTALL, cwd=where, run_as_root=True)
class ProgramRuntime(ComponentBase):
def __init__(self, component_name, *args, **kargs):
ComponentBase.__init__(self, component_name, *args, **kargs)
self.tracereader = tr.TraceReader(self.tracedir, tr.IN_TRACE)
self.tracewriter = tr.TraceWriter(self.tracedir, tr.START_TRACE)
self.starttracereader = tr.TraceReader(self.tracedir, tr.START_TRACE)
self.tracewriter = tr.TraceWriter(tr.trace_fn(self.tracedir, tr.START_TRACE))
self.tracereader = tr.TraceReader(tr.trace_fn(self.tracedir, tr.START_TRACE))
def _get_apps_to_start(self):
return list()
@ -457,20 +447,17 @@ class ProgramRuntime(ComponentBase):
return am_started
def _locate_killers(self):
start_traces = self.starttracereader.apps_started()
start_traces = self.tracereader.apps_started()
killer_instances = dict()
to_kill = list()
for mp in start_traces:
fn = mp['trace_fn']
app_name = mp['name']
for (trace_fn, app_name) in start_traces:
# Figure out which class will stop it
killcls = None
runtype = "??"
for (cmd, action) in tr.parse_fn(fn):
for (cmd, action) in tr.TraceReader(trace_fn).read():
if cmd == settings.RUN_TYPE_TYPE and action:
runtype = action
killcls = RUNNER_CLS_MAPPING.get(runtype)
killcls = RUNNER_CLS_MAPPING.get(action)
break
# Did we find a class that can do it?
if killcls:
if killcls in killer_instances:
@ -480,7 +467,7 @@ class ProgramRuntime(ComponentBase):
killer_instances[killcls] = killer
to_kill.append((app_name, killer))
else:
msg = "Could not figure out which class to use to stop (%s, %s) of run type (%s)" % (app_name, fn, runtype)
msg = "Could not figure out which class to use to stop (%s, %s)" % (app_name, trace_fn)
raise excp.StopException(msg)
return to_kill
@ -488,8 +475,8 @@ class ProgramRuntime(ComponentBase):
to_kill = self._locate_killers()
for (app_name, killer) in to_kill:
killer.stop(app_name)
LOG.debug("Deleting start trace file [%s]" % (self.starttracereader.trace_fn))
sh.unlink(self.starttracereader.trace_fn)
LOG.debug("Deleting start trace file [%s]" % (self.tracereader.filename()))
sh.unlink(self.tracereader.filename())
return len(to_kill)
def status(self):

12
devstack/components/glance.py
View File

@ -150,7 +150,7 @@ class GlanceInstaller(comp.PythonInstallComponent):
"(and is empty)" % (cache_dir))
#destroy then recreate the image cache directory
sh.deldir(cache_dir)
self.tracewriter.make_dir(cache_dir)
self.tracewriter.dirs_made(*sh.mkdirslist(cache_dir))
if config.get('default', 'default_store') == 'file':
file_dir = config.get('default', 'filesystem_store_datadir')
if file_dir:
@ -158,26 +158,24 @@ class GlanceInstaller(comp.PythonInstallComponent):
#delete existing images
#and recreate the image directory
sh.deldir(file_dir)
self.tracewriter.make_dir(file_dir)
self.tracewriter.dirs_made(*sh.mkdirslist(file_dir))
log_filename = config.get('default', 'log_file')
if log_filename:
LOG.info("Ensuring log file %s exists and is empty." % (log_filename))
log_dir = sh.dirname(log_filename)
if log_dir:
LOG.info("Ensuring log directory %s exists." % (log_dir))
self.tracewriter.make_dir(log_dir)
self.tracewriter.dirs_made(*sh.mkdirslist(log_dir))
#destroy then recreate it (the log file)
sh.unlink(log_filename)
sh.touch_file(log_filename)
self.tracewriter.file_touched(log_filename)
self.tracewriter.file_touched(sh.touch_file(log_filename))
if config.getboolean('default', 'delayed_delete'):
data_dir = config.get('default', 'scrubber_datadir')
if data_dir:
LOG.info("Ensuring scrubber data dir %s exists and is empty." % (data_dir))
#destroy then recreate the scrubber data directory
sh.deldir(data_dir)
self.tracewriter.make_dir(data_dir)
#we might need to handle more in the future...
self.tracewriter.dirs_made(*sh.mkdirslist(data_dir))
#nothing modified so just return the original
return contents

13
devstack/components/horizon.py
View File

@ -76,6 +76,9 @@ APACHE_FIXUPS = {
}
APACHE_FIXUPS_DISTROS = [settings.RHEL6]
#for when quantum client is not need we need some fake files so python doesn't croak
FAKE_QUANTUM_FILES = ['__init__.py', 'client.py']
#users which apache may not like starting as
BAD_APACHE_USERS = ['root']
@ -153,7 +156,7 @@ class HorizonInstaller(comp.PythonInstallComponent):
def _setup_blackhole(self):
#create an empty directory that apache uses as docroot
self.tracewriter.make_dir(sh.joinpths(self.appdir, BLACKHOLE_DIR))
self.tracewriter.dirs_made(*sh.mkdirslist(sh.joinpths(self.appdir, BLACKHOLE_DIR)))
def _sync_db(self):
#Initialize the horizon database (it stores sessions and notices shown to users).
@ -175,7 +178,7 @@ class HorizonInstaller(comp.PythonInstallComponent):
def pre_install(self):
comp.PythonInstallComponent.pre_install(self)
self.tracewriter.make_dir(self.log_dir)
self.tracewriter.dirs_made(*sh.mkdirslist(self.log_dir))
def _config_fixups(self):
#currently just handling rhel fixups
@ -209,9 +212,9 @@ class HorizonInstaller(comp.PythonInstallComponent):
#TODO remove this...
quantum_dir = sh.joinpths(self.dash_dir, 'quantum')
if not sh.isdir(quantum_dir):
self.tracewriter.make_dir(quantum_dir)
self.tracewriter.touch_file(sh.joinpths(quantum_dir, '__init__.py'))
self.tracewriter.touch_file(sh.joinpths(quantum_dir, 'client.py'))
self.tracewriter.dirs_made(*sh.mkdirslist(quantum_dir))
for fn in FAKE_QUANTUM_FILES:
self.tracewriter.file_touched(sh.touch_file(sh.joinpths(quantum_dir, fn)))
def post_install(self):
comp.PythonInstallComponent.post_install(self)

5
devstack/components/keystone.py
View File

@ -160,11 +160,10 @@ class KeystoneInstaller(comp.PythonInstallComponent):
log_dir = sh.dirname(log_filename)
if log_dir:
LOG.info("Ensuring log directory %s exists." % (log_dir))
self.tracewriter.make_dir(log_dir)
self.tracewriter.dirs_made(*sh.mkdirslist(log_dir))
#destroy then recreate it (the log file)
sh.unlink(log_filename)
sh.touch_file(log_filename)
self.tracewriter.file_touched(log_filename)
self.tracewriter.file_touched(sh.touch_file(log_filename))
elif name == CATALOG_CONF:
nlines = list()
if utils.service_enabled(settings.SWIFT, self.instances):

20
devstack/components/nova.py
View File

@ -379,13 +379,12 @@ class NovaInstaller(comp.PythonInstallComponent):
def _generate_nova_conf(self):
LOG.info("Generating dynamic content for nova configuration (%s)." % (API_CONF))
conf_gen = NovaConfConfigurator(self)
nova_conf = conf_gen.configure()
tgtfn = self._get_target_config_name(API_CONF)
LOG.info("Writing nova configuration to %s" % (tgtfn))
LOG.debug(nova_conf)
self.tracewriter.make_dir(sh.dirname(tgtfn))
sh.write_file(tgtfn, nova_conf)
self.tracewriter.cfg_write(tgtfn)
nova_conf_contents = conf_gen.configure()
conf_fn = self._get_target_config_name(API_CONF)
LOG.info("Writing nova configuration to %s" % (conf_fn))
LOG.debug(nova_conf_contents)
self.tracewriter.dirs_made(*sh.mkdirslist(sh.dirname(conf_fn)))
self.tracewriter.cfg_file_written(sh.write_file(conf_fn, nova_conf_contents))
def _get_source_config(self, config_fn):
name = config_fn
@ -419,11 +418,12 @@ class NovaInstaller(comp.PythonInstallComponent):
# TODO: maybe this should be a subclass that handles these differences
driver_canon = _canon_virt_driver(self.cfg.get('nova', 'virt_driver'))
if (self.distro in POLICY_DISTROS) and driver_canon == virsh.VIRT_TYPE:
dirs_made = list()
with sh.Rooted(True):
dirsmade = sh.mkdirslist(sh.dirname(LIBVIRT_POLICY_FN))
dirs_made = sh.mkdirslist(sh.dirname(LIBVIRT_POLICY_FN))
sh.write_file(LIBVIRT_POLICY_FN, LIBVIRT_POLICY_CONTENTS)
self.tracewriter.dir_made(*dirsmade)
self.tracewriter.cfg_write(LIBVIRT_POLICY_FN)
self.tracewriter.dirs_made(*dirs_made)
self.tracewriter.cfg_file_written(LIBVIRT_POLICY_FN)
configs_made += 1
return configs_made

15
devstack/components/swift.py
View File

@ -167,20 +167,23 @@ class SwiftInstaller(comp.PythonInstallComponent):
def _create_nodes(self):
for i in range(1, 5):
self.tracewriter.make_dir(sh.joinpths(self.fs_dev, '%d/node' % i))
self.tracewriter.symlink(sh.joinpths(self.fs_dev, str(i)),
sh.joinpths(self.datadir, str(i)))
self.tracewriter.dirs_made(sh.mkdirslist(sh.joinpths(self.fs_dev, '%d/node' % i)))
link_tgt = sh.joinpths(self.datadir, str(i))
sh.symlink(sh.joinpths(self.fs_dev, str(i)), link_tgt)
self.tracewriter.symlink_made(link_tgt)
start_port = (6010 + (i - 1) * 5)
self._create_node_config(i, start_port)
self._delete_templates()
def _turn_on_rsync(self):
self.tracewriter.symlink(sh.joinpths(self.cfgdir, RSYNC_CONF), RSYNCD_CONF_LOC)
sh.symlink(sh.joinpths(self.cfgdir, RSYNC_CONF), RSYNCD_CONF_LOC)
self.tracewriter.symlink_made(RSYNCD_CONF_LOC)
sh.replace_in(RSYNC_CONF_LOC, RSYNC_ON_OFF_RE, 'RSYNC_ENABLE=true', True)
def _create_log_dirs(self):
self.tracewriter.make_dir(sh.joinpths(self.logdir, 'hourly'))
self.tracewriter.symlink(sh.joinpths(self.cfgdir, SYSLOG_CONF), SWIFT_RSYNC_LOC)
self.tracewriter.dirs_made(*sh.mkdirslist(sh.joinpths(self.logdir, 'hourly')))
sh.symlink(sh.joinpths(self.cfgdir, SYSLOG_CONF), SWIFT_RSYNC_LOC)
self.tracewriter.symlink_made(SWIFT_RSYNC_LOC)
def _setup_binaries(self):
sh.move(sh.joinpths(self.cfgdir, SWIFT_MAKERINGS), self.makerings_file)

7
devstack/runners/fork.py
View File

@ -165,11 +165,10 @@ class ForkRunner(base.RunnerBase):
os._exit(0)
def _do_trace(self, fn, kvs):
tracefn = tr.touch_trace(self.trace_dir, fn)
runtrace = tr.Trace(tracefn)
run_trace = tr.TraceWriter(tr.trace_fn(self.trace_dir, fn))
for (k, v) in kvs.items():
runtrace.trace(k, v)
return tracefn
run_trace.trace(k, v)
return run_trace.filename()
def start(self, app_name, runtime_info):
(program, appdir, program_args) = runtime_info

19
devstack/runners/screen.py
View File

@ -78,15 +78,14 @@ class ScreenRunner(base.RunnerBase):
self.socket_dir = sh.joinpths(tempfile.gettempdir(), SCREEN_SOCKET_DIR_NAME)
def stop(self, app_name):
fn_name = SCREEN_TEMPL % (app_name)
trace_fn = tr.trace_fn(self.trace_dir, fn_name)
trace_fn = tr.trace_fn(self.trace_dir, SCREEN_TEMPL % (app_name))
session_id = self._find_session(app_name, trace_fn)
self._do_stop(app_name, session_id)
sh.unlink(trace_fn)
def _find_session(self, app_name, trace_fn):
session_id = None
for (key, value) in tr.parse_fn(trace_fn):
for (key, value) in tr.TraceReader(trace_fn).read():
if key == SESSION_ID and value:
session_id = value
if not session_id:
@ -200,12 +199,10 @@ class ScreenRunner(base.RunnerBase):
sh.chmod(d, perm)
def _begin_start(self, name, program, args):
fn_name = SCREEN_TEMPL % (name)
tracefn = tr.touch_trace(self.trace_dir, fn_name)
runtrace = tr.Trace(tracefn)
runtrace.trace(TYPE, RUN_TYPE)
runtrace.trace(NAME, name)
runtrace.trace(ARGS, json.dumps(args))
run_trace = tr.TraceWriter(tr.trace_fn(self.trace_dir, SCREEN_TEMPL % (name)))
run_trace.trace(TYPE, RUN_TYPE)
run_trace.trace(NAME, name)
run_trace.trace(ARGS, json.dumps(args))
full_cmd = [program] + list(args)
session_name = self._get_session()
inited_screen = False
@ -216,7 +213,7 @@ class ScreenRunner(base.RunnerBase):
if session_name is None:
msg = "After initializing screen with session named [%s], no screen session with that name was found!" % (SESSION_NAME)
raise excp.StartException(msg)
runtrace.trace(SESSION_ID, session_name)
run_trace.trace(SESSION_ID, session_name)
if inited_screen or not sh.isfile(SCREEN_RC):
rc_gen = ScreenRcGenerator(self)
rc_contents = rc_gen.create(session_name, self._get_env())
@ -224,7 +221,7 @@ class ScreenRunner(base.RunnerBase):
LOG.info("Writing your created screen rc file to [%s]" % (out_fn))
sh.write_file(out_fn, rc_contents)
self._do_start(session_name, name, full_cmd)
return tracefn
return run_trace.filename()
def start(self, app_name, runtime_info):
(program, _, program_args) = runtime_info

12
devstack/runners/upstart.py
View File

@ -116,12 +116,10 @@ class UpstartRunner(base.RunnerBase):
sh.chmod(cfg_fn, 0666)
def _start(self, app_name, program, program_args):
fn_name = UPSTART_TEMPL % (app_name)
tracefn = tr.touch_trace(self.trace_dir, fn_name)
runtrace = tr.Trace(tracefn)
runtrace.trace(TYPE, RUN_TYPE)
runtrace.trace(NAME, app_name)
runtrace.trace(ARGS, json.dumps(program_args))
run_trace = tr.TraceWriter(tr.trace_fn(self.trace_dir, UPSTART_TEMPL % (app_name)))
run_trace.trace(TYPE, RUN_TYPE)
run_trace.trace(NAME, app_name)
run_trace.trace(ARGS, json.dumps(program_args))
# Emit the start, keep track and only do one per component name
component_event = self.component_name + START_EVENT_SUFFIX
if component_event in self.events:
@ -131,7 +129,7 @@ class UpstartRunner(base.RunnerBase):
cmd = EMIT_BASE_CMD + [component_event]
sh.execute(*cmd, run_as_root=True)
self.events.add(component_event)
return tracefn
return run_trace.filename()
def start(self, app_name, runtime_info):
(program, _, program_args) = runtime_info

3
devstack/shell.py
View File

@ -348,6 +348,7 @@ def append_file(fn, text, flush=True, quiet=False):
f.write(text)
if flush:
f.flush()
return fn
def write_file(fn, text, flush=True, quiet=False):
@ -357,6 +358,7 @@ def write_file(fn, text, flush=True, quiet=False):
f.write(text)
if flush:
f.flush()
return fn
def touch_file(fn, die_if_there=True, quiet=False, file_size=0):
@ -369,6 +371,7 @@ def touch_file(fn, die_if_there=True, quiet=False, file_size=0):
if die_if_there:
msg = "Can not touch & truncate file %s since it already exists" % (fn)
raise excp.FileException(msg)
return fn
def load_file(fn, quiet=False):

261
devstack/trace.py
View File

@ -34,7 +34,6 @@ FILE_TOUCHED = "FILE_TOUCHED"
DOWNLOADED = "DOWNLOADED"
AP_STARTED = "AP_STARTED"
PIP_INSTALL = 'PIP_INSTALL'
EXEC_CMD = 'EXEC_CMD'
#trace file types
PY_TRACE = "python"
@ -46,153 +45,149 @@ TRACE_VERSION = "TRACE_VERSION"
TRACE_VER = 0x1
class Trace(object):
def __init__(self, tracefn):
self.tracefn = tracefn
def trace_fn(root_dir, name):
return sh.joinpths(root_dir, name + TRACE_EXT)
def filename(self):
return self.tracefn
class TraceWriter(object):
def __init__(self, trace_filename):
self.trace_fn = trace_filename
self.started = False
def trace(self, cmd, action=None):
if action is None:
action = date.rcf8222date()
line = TRACE_FMT % (cmd, action)
sh.append_file(self.tracefn, line)
if cmd is not None:
sh.append_file(self.trace_fn, TRACE_FMT % (cmd, action))
class TraceWriter(object):
def __init__(self, root, name):
self.tracer = None
self.root = root
self.name = name
self.filename = None
self.started = False
def filename(self):
return self.trace_fn
def _start(self):
if self.started:
return
else:
dirs = sh.mkdirslist(self.root)
self.filename = touch_trace(self.root, self.name)
self.tracer = Trace(self.filename)
self.tracer.trace(TRACE_VERSION, str(TRACE_VER))
if dirs:
for d in dirs:
self.tracer.trace(DIR_MADE, d)
trace_dirs = sh.mkdirslist(sh.dirname(self.trace_fn))
sh.touch_file(self.trace_fn)
self.trace(TRACE_VERSION, str(TRACE_VER))
self.started = True
self.dirs_made(*trace_dirs)
def py_install(self, name, trace_filename, where):
def py_installed(self, name, where):
self._start()
what = dict()
what['name'] = name
what['trace'] = trace_filename
what['where'] = where
self.tracer.trace(PYTHON_INSTALL, json.dumps(what))
self.trace(PYTHON_INSTALL, json.dumps(what))
def cfg_write(self, cfgfile):
def cfg_file_written(self, fn):
self._start()
self.tracer.trace(CFG_WRITING_FILE, cfgfile)
self.trace(CFG_WRITING_FILE, fn)
def symlink(self, source, link):
def symlink_made(self, link):
self._start()
dirs = sh.symlink(source, link)
self.dir_made(*dirs)
self.tracer.trace(SYMLINK_MAKE, link)
self.trace(SYMLINK_MAKE, link)
def downloaded(self, tgt, fromwhere):
def download_happened(self, tgt, uri):
self._start()
what = dict()
what['target'] = tgt
what['from'] = fromwhere
self.tracer.trace(DOWNLOADED, json.dumps(what))
what['from'] = uri
self.trace(DOWNLOADED, json.dumps(what))
def pip_install(self, name, pip_info):
def pip_installed(self, name, pip_info):
self._start()
what = dict()
what['name'] = name
what['pip_meta'] = pip_info
self.tracer.trace(PIP_INSTALL, json.dumps(what))
self.trace(PIP_INSTALL, json.dumps(what))
def make_dir(self, path):
self._start()
dirs = sh.mkdirslist(path)
self.dir_made(*dirs)
return path
def touch_file(self, path):
self._start()
sh.touch_file(path)
self.file_touched(path)
return path
def dir_made(self, *dirs):
def dirs_made(self, *dirs):
self._start()
for d in dirs:
self.tracer.trace(DIR_MADE, d)
self.trace(DIR_MADE, d)
def file_touched(self, fn):
self._start()
self.tracer.trace(FILE_TOUCHED, fn)
self.trace(FILE_TOUCHED, fn)
def package_install(self, name, pkg_info):
def package_installed(self, name, pkg_info):
self._start()
what = dict()
what['name'] = name
what['pkg_meta'] = pkg_info
self.tracer.trace(PKG_INSTALL, json.dumps(what))
self.trace(PKG_INSTALL, json.dumps(what))
def started_info(self, name, info_fn):
self._start()
data = dict()
data['name'] = name
data['trace_fn'] = info_fn
self.tracer.trace(AP_STARTED, json.dumps(data))
def exec_cmd(self, cmd, result):
self._start()
data = dict()
data['cmd'] = cmd
data['result'] = result
self.tracer.trace(EXEC_CMD, json.dumps(data))
self.trace(AP_STARTED, json.dumps(data))
class TraceReader(object):
def __init__(self, root, name):
self.root = root
self.name = name
self.trace_fn = trace_fn(root, name)
def __init__(self, trace_filename):
self.trace_fn = trace_filename
self.contents = None
def _readpy(self):
lines = self._read()
pyentries = list()
for (cmd, action) in lines:
if cmd == PYTHON_INSTALL and len(action):
jentry = json.loads(action)
if type(jentry) is dict:
pyentries.append(jentry)
return pyentries
def filename(self):
return self.trace_fn
def _read(self):
return parse_name(self.root, self.name)
def _parse(self):
fn = self.trace_fn
if not sh.isfile(fn):
msg = "No trace found at filename %s" % (fn)
raise excp.NoTraceException(msg)
contents = sh.load_file(fn)
lines = contents.splitlines()
accum = list()
for line in lines:
ep = self._split_line(line)
if ep is None:
continue
accum.append(tuple(ep))
return accum
def read(self):
if self.contents is None:
self.contents = self._parse()
return self.contents
def _split_line(self, line):
pieces = line.split("-", 1)
if len(pieces) == 2:
cmd = pieces[0].rstrip()
action = pieces[1].lstrip()
return (cmd, action)
else:
return None
def exists(self):
return sh.exists(self.trace_fn)
def downloaded(self):
lines = self._read()
def py_listing(self):
lines = self.read()
py_entries = list()
for (cmd, action) in lines:
if cmd == PYTHON_INSTALL and len(action):
entry = json.loads(action)
if type(entry) is dict:
py_entries.append((entry.get("name"), entry.get("where")))
return py_entries
def download_locations(self):
lines = self.read()
locs = list()
for (cmd, action) in lines:
if cmd == DOWNLOADED and len(action):
jentry = json.loads(action)
if type(jentry) is dict:
locs.append(jentry)
entry = json.loads(action)
if type(entry) is dict:
locs.append(entry.get('target'))
return locs
def py_listing(self):
return self._readpy()
def files_touched(self):
lines = self._read()
lines = self.read()
files = list()
for (cmd, action) in lines:
if cmd == FILE_TOUCHED and len(action):
@ -202,7 +197,7 @@ class TraceReader(object):
return files
def dirs_made(self):
lines = self._read()
lines = self.read()
dirs = list()
for (cmd, action) in lines:
if cmd == DIR_MADE and len(action):
@ -214,17 +209,17 @@ class TraceReader(object):
return dirs
def apps_started(self):
lines = self._read()
files = list()
lines = self.read()
app_info = list()
for (cmd, action) in lines:
if cmd == AP_STARTED and len(action):
jdec = json.loads(action)
if type(jdec) is dict:
files.append(jdec)
return files
entry = json.loads(action)
if type(entry) is dict:
app_info.append((entry.get('trace_fn'), entry.get('name')))
return app_info
def symlinks_made(self):
lines = self._read()
lines = self.read()
files = list()
for (cmd, action) in lines:
if cmd == SYMLINK_MAKE and len(action):
@ -235,7 +230,7 @@ class TraceReader(object):
return files
def files_configured(self):
lines = self._read()
lines = self.read()
files = list()
for (cmd, action) in lines:
if cmd == CFG_WRITING_FILE and len(action):
@ -245,79 +240,31 @@ class TraceReader(object):
return files
def pips_installed(self):
lines = self._read()
pipsinstalled = dict()
lines = self.read()
pips_installed = dict()
pip_list = list()
for (cmd, action) in lines:
if cmd == PIP_INSTALL and len(action):
pip_list.append(action)
for pdata in pip_list:
pip_info_full = json.loads(pdata)
for pip_data in pip_list:
pip_info_full = json.loads(pip_data)
if type(pip_info_full) is dict:
name = pip_info_full.get('name')
if name and len(name):
pipsinstalled[name] = pip_info_full.get('pip_meta')
return pipsinstalled
if name:
pips_installed[name] = pip_info_full.get('pip_meta')
return pips_installed
def packages_installed(self):
lines = self._read()
pkgsinstalled = dict()
lines = self.read()
pkgs_installed = dict()
pkg_list = list()
for (cmd, action) in lines:
if cmd == PKG_INSTALL and len(action):
pkg_list.append(action)
for pdata in pkg_list:
pkg_info = json.loads(pdata)
for pkg_data in pkg_list:
pkg_info = json.loads(pkg_data)
if type(pkg_info) is dict:
name = pkg_info.get('name')
if name and len(name):
pkgsinstalled[name] = pkg_info.get('pkg_meta')
return pkgsinstalled
def trace_fn(rootdir, name):
fullname = name + TRACE_EXT
return sh.joinpths(rootdir, fullname)
def touch_trace(rootdir, name):
tracefn = trace_fn(rootdir, name)
sh.touch_file(tracefn)
return tracefn
def split_line(line):
pieces = line.split("-", 1)
if len(pieces) == 2:
cmd = pieces[0].rstrip()
action = pieces[1].lstrip()
return (cmd, action)
else:
return None
def read(rootdir, name):
pth = trace_fn(rootdir, name)
contents = sh.load_file(pth)
lines = contents.splitlines()
return lines
def parse_fn(fn):
if not sh.isfile(fn):
msg = "No trace found at filename %s" % (fn)
raise excp.NoTraceException(msg)
contents = sh.load_file(fn)
lines = contents.splitlines()
accum = list()
for line in lines:
ep = split_line(line)
if ep is None:
continue
accum.append(tuple(ep))
return accum
def parse_name(rootdir, name):
fn = trace_fn(rootdir, name)
return parse_fn(fn)
if name:
pkgs_installed[name] = pkg_info.get('pkg_meta')
return pkgs_installed

30
stack
View File

@ -18,7 +18,6 @@
import logging
import logging.config
import os
import sys
import time
import traceback
@ -105,6 +104,23 @@ def run(args):
return True
def configure_logging(args):
# Debug by default
root_logger = logging.getLogger('')
root_logger.setLevel(logging.DEBUG)
# Set our pretty logger
console_logger = logging.StreamHandler(sys.stdout)
console_format = '%(levelname)s: @%(name)s : %(message)s'
console_logger.setFormatter(colorlog.TermFormatter(console_format))
root_logger.addHandler(console_logger)
# Adjust logging verbose level based on the command line switch.
log_level = logging.DEBUG if args['verbosity'] >= 2 else logging.INFO
root_logger.setLevel(log_level)
def main():
#do this first so people can see the help message...
@ -112,16 +128,8 @@ def main():
prog_name = sys.argv[0]
# Configure logging
root_logger = logging.getLogger('')
root_logger.setLevel(logging.DEBUG)
console = logging.StreamHandler(sys.stdout)
console_format = '%(levelname)s: @%(name)s : %(message)s'
console.setFormatter(colorlog.TermFormatter(console_format))
root_logger.addHandler(console)
# Adjust logging verbose level based on the command line switch.
log_level = logging.DEBUG if args['verbosity'] >= 2 else logging.INFO
root_logger.setLevel(log_level)
configure_logging(args)
LOG.debug("Command line options %s" % (args))
#will need root to setup openstack