x/anvil
Code Issues Proposed changes

Cleanup aisle 10

Browse Source
This commit is contained in:
Joshua Harlow 2012-03-08 18:17:05 -08:00
parent e59151182c
commit cd17a83a60
12 changed files with 215 additions and 273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
devstack/component.py
View File

@ -77,16 +77,16 @@ class ComponentBase(object):
pass pass
def is_started(self): def is_started(self):
return tr.TraceReader(self.tracedir, tr.START_TRACE).exists() return tr.TraceReader(tr.trace_fn(self.tracedir, tr.START_TRACE)).exists()
def is_installed(self): def is_installed(self):
return tr.TraceReader(self.tracedir, tr.IN_TRACE).exists() return tr.TraceReader(tr.trace_fn(self.tracedir, tr.IN_TRACE)).exists()
class PkgInstallComponent(ComponentBase): class PkgInstallComponent(ComponentBase):
def __init__(self, component_name, *args, **kargs): def __init__(self, component_name, *args, **kargs):
ComponentBase.__init__(self, component_name, *args, **kargs) ComponentBase.__init__(self, component_name, *args, **kargs)
self.tracewriter = tr.TraceWriter(self.tracedir, tr.IN_TRACE) self.tracewriter = tr.TraceWriter(tr.trace_fn(self.tracedir, tr.IN_TRACE))
def _get_download_locations(self): def _get_download_locations(self):
return list() return list()
@ -109,7 +109,7 @@ class PkgInstallComponent(ComponentBase):
branch = self.cfg.get(cfg_section, cfg_key) branch = self.cfg.get(cfg_section, cfg_key)
(cfg_section, cfg_key) = uri_tuple (cfg_section, cfg_key) = uri_tuple
uri = self.cfg.get(cfg_section, cfg_key) uri = self.cfg.get(cfg_section, cfg_key)
self.tracewriter.downloaded(target_loc, uri) self.tracewriter.download_happened(target_loc, uri)
dirs_made = down.download(target_loc, uri, branch) dirs_made = down.download(target_loc, uri, branch)
#ensure this is always added so that #ensure this is always added so that
#if a keep old happens then this of course #if a keep old happens then this of course
@ -117,7 +117,7 @@ class PkgInstallComponent(ComponentBase):
#then this won't be deleted this time around #then this won't be deleted this time around
#adding it in is harmless and willl make sure its removed #adding it in is harmless and willl make sure its removed
dirs_made.append(target_loc) dirs_made.append(target_loc)
self.tracewriter.dir_made(*dirs_made) self.tracewriter.dirs_made(*dirs_made)
return len(locations) return len(locations)
def _get_param_map(self, config_fn): def _get_param_map(self, config_fn):
@ -143,7 +143,7 @@ class PkgInstallComponent(ComponentBase):
LOG.info("Installing packages (%s)." % (", ".join(pkgnames))) LOG.info("Installing packages (%s)." % (", ".join(pkgnames)))
#do this before install just incase it craps out half way through #do this before install just incase it craps out half way through
for name in pkgnames: for name in pkgnames:
self.tracewriter.package_install(name, pkgs.get(name)) self.tracewriter.package_installed(name, pkgs.get(name))
#now actually install #now actually install
self.packager.install_batch(pkgs) self.packager.install_batch(pkgs)
return self.tracedir return self.tracedir
@ -189,22 +189,20 @@ class PkgInstallComponent(ComponentBase):
for fn in configs: for fn in configs:
#get the params and where it should come from and where it should go #get the params and where it should come from and where it should go
parameters = self._get_param_map(fn) parameters = self._get_param_map(fn)
tgtfn = self._get_target_config_name(fn) tgt_fn = self._get_target_config_name(fn)
#ensure directory is there (if not created previously) #ensure directory is there (if not created previously)
self.tracewriter.make_dir(sh.dirname(tgtfn)) self.tracewriter.dirs_made(*sh.mkdirslist(sh.dirname(tgt_fn)))
#now configure it #now configure it
LOG.info("Configuring file %s" % (fn)) LOG.info("Configuring file %s" % (fn))
(sourcefn, contents) = self._get_source_config(fn) (source_fn, contents) = self._get_source_config(fn)
LOG.debug("Replacing parameters in file %s" % (sourcefn)) LOG.debug("Replacing parameters in file %s" % (source_fn))
LOG.debug("Replacements = %s" % (parameters)) LOG.debug("Replacements = %s" % (parameters))
contents = utils.param_replace(contents, parameters) contents = utils.param_replace(contents, parameters)
LOG.debug("Applying side-effects of param replacement for template %s" % (sourcefn)) LOG.debug("Applying side-effects of param replacement for template %s" % (source_fn))
contents = self._config_adjust(contents, fn) contents = self._config_adjust(contents, fn)
LOG.info("Writing configuration file %s" % (tgtfn)) LOG.info("Writing configuration file %s" % (tgt_fn))
#this trace is used to remove the files configured #this trace is used to remove the files configured
#do this before write just incase it craps out half way through self.tracewriter.cfg_file_written(sh.write_file(tgt_fn, contents))
self.tracewriter.cfg_write(tgtfn)
sh.write_file(tgtfn, contents)
return len(configs) return len(configs)
def _configure_symlinks(self): def _configure_symlinks(self):
@ -215,7 +213,8 @@ class PkgInstallComponent(ComponentBase):
link = links.get(source) link = links.get(source)
try: try:
LOG.info("Symlinking %s => %s" % (link, source)) LOG.info("Symlinking %s => %s" % (link, source))
self.tracewriter.symlink(source, link) self.tracewriter.dirs_made(*sh.symlink(source, link))
self.tracewriter.symlink_made(link)
except OSError: except OSError:
LOG.warn("Symlink %s => %s already exists." % (link, source)) LOG.warn("Symlink %s => %s already exists." % (link, source))
return len(links) return len(links)
@ -254,31 +253,28 @@ class PythonInstallComponent(PkgInstallComponent):
LOG.info("Setting up %s pips (%s)" % (len(pips), ", ".join(pips.keys()))) LOG.info("Setting up %s pips (%s)" % (len(pips), ", ".join(pips.keys())))
#do this before install just incase it craps out half way through #do this before install just incase it craps out half way through
for name in pips.keys(): for name in pips.keys():
self.tracewriter.pip_install(name, pips.get(name)) self.tracewriter.pip_installed(name, pips.get(name))
#now install #now install
pip.install(pips, self.distro) pip.install(pips, self.distro)
def _format_stderr_out(self, stderr, stdout):
combined = ["===STDOUT===", str(stdout), "===STDERR===", str(stderr)]
return utils.joinlinesep(*combined)
def _format_trace_name(self, name):
return "%s-%s" % (tr.PY_TRACE, name)
def _install_python_setups(self): def _install_python_setups(self):
pydirs = self._get_python_directories() pydirs = self._get_python_directories()
if pydirs: if pydirs:
LOG.info("Setting up %s python directories (%s)" % (len(pydirs), pydirs)) LOG.info("Setting up %s python directories (%s)" % (len(pydirs), pydirs))
for (name, wkdir) in pydirs.items(): for (name, wkdir) in pydirs.items():
working_dir = wkdir or self.appdir working_dir = wkdir or self.appdir
self.tracewriter.make_dir(working_dir) #ensure working dir is there
record_fn = tr.touch_trace(self.tracedir, self._format_trace_name(name)) self.tracewriter.dirs_made(*sh.mkdirslist(working_dir))
#do this before write just incase it craps out half way through #do this before write just incase it craps out half way through
self.tracewriter.file_touched(record_fn) self.tracewriter.py_installed(name, working_dir)
self.tracewriter.py_install(name, record_fn, working_dir)
#now actually do it #now actually do it
(stdout, stderr) = sh.execute(*PY_INSTALL, cwd=working_dir, run_as_root=True) (stdout, stderr) = sh.execute(*PY_INSTALL, cwd=working_dir, run_as_root=True)
sh.write_file(record_fn, self._format_stderr_out(stderr, stdout)) py_trace_name = "%s-%s" % (tr.PY_TRACE, name)
py_writer = tr.TraceWriter(tr.trace_fn(self.tracedir, py_trace_name))
py_writer.trace("CMD", " ".join(PY_INSTALL))
py_writer.trace("STDOUT", stdout)
py_writer.trace("STDERR", stderr)
self.tracewriter.file_touched(py_writer.filename())
def _python_install(self): def _python_install(self):
self._install_pips() self._install_pips()
@ -293,7 +289,7 @@ class PythonInstallComponent(PkgInstallComponent):
class PkgUninstallComponent(ComponentBase): class PkgUninstallComponent(ComponentBase):
def __init__(self, component_name, *args, **kargs): def __init__(self, component_name, *args, **kargs):
ComponentBase.__init__(self, component_name, *args, **kargs) ComponentBase.__init__(self, component_name, *args, **kargs)
self.tracereader = tr.TraceReader(self.tracedir, tr.IN_TRACE) self.tracereader = tr.TraceReader(tr.trace_fn(self.tracedir, tr.IN_TRACE))
self.keep_old = kargs.get("keep_old") self.keep_old = kargs.get("keep_old")
def unconfigure(self): def unconfigure(self):
@ -356,12 +352,8 @@ class PkgUninstallComponent(ComponentBase):
if dirsmade: if dirsmade:
dirsmade = [sh.abspth(d) for d in dirsmade] dirsmade = [sh.abspth(d) for d in dirsmade]
if self.keep_old: if self.keep_old:
downloads = (self.tracereader.downloaded()) download_locations = self.tracereader.download_locations()
places = set() places = set(download_locations)
for info in downloads:
download_place = info.get('target')
if download_place:
places.add(download_place)
LOG.info("Keeping %s download directories [%s]" % (len(places), ",".join(sorted(places)))) LOG.info("Keeping %s download directories [%s]" % (len(places), ",".join(sorted(places))))
for download_place in places: for download_place in places:
dirsmade = sh.remove_parents(download_place, dirsmade) dirsmade = sh.remove_parents(download_place, dirsmade)
@ -389,17 +381,15 @@ class PythonUninstallComponent(PkgUninstallComponent):
pylisting = self.tracereader.py_listing() pylisting = self.tracereader.py_listing()
if pylisting: if pylisting:
LOG.info("Uninstalling %s python setups." % (len(pylisting))) LOG.info("Uninstalling %s python setups." % (len(pylisting)))
for entry in pylisting: for (_, where) in pylisting:
where = entry.get('where')
sh.execute(*PY_UNINSTALL, cwd=where, run_as_root=True) sh.execute(*PY_UNINSTALL, cwd=where, run_as_root=True)
class ProgramRuntime(ComponentBase): class ProgramRuntime(ComponentBase):
def __init__(self, component_name, *args, **kargs): def __init__(self, component_name, *args, **kargs):
ComponentBase.__init__(self, component_name, *args, **kargs) ComponentBase.__init__(self, component_name, *args, **kargs)
self.tracereader = tr.TraceReader(self.tracedir, tr.IN_TRACE) self.tracewriter = tr.TraceWriter(tr.trace_fn(self.tracedir, tr.START_TRACE))
self.tracewriter = tr.TraceWriter(self.tracedir, tr.START_TRACE) self.tracereader = tr.TraceReader(tr.trace_fn(self.tracedir, tr.START_TRACE))
self.starttracereader = tr.TraceReader(self.tracedir, tr.START_TRACE)
def _get_apps_to_start(self): def _get_apps_to_start(self):
return list() return list()
@ -457,20 +447,17 @@ class ProgramRuntime(ComponentBase):
return am_started return am_started
def _locate_killers(self): def _locate_killers(self):
start_traces = self.starttracereader.apps_started() start_traces = self.tracereader.apps_started()
killer_instances = dict() killer_instances = dict()
to_kill = list() to_kill = list()
for mp in start_traces: for (trace_fn, app_name) in start_traces:
fn = mp['trace_fn']
app_name = mp['name']
# Figure out which class will stop it # Figure out which class will stop it
killcls = None killcls = None
runtype = "??" for (cmd, action) in tr.TraceReader(trace_fn).read():
for (cmd, action) in tr.parse_fn(fn):
if cmd == settings.RUN_TYPE_TYPE and action: if cmd == settings.RUN_TYPE_TYPE and action:
runtype = action killcls = RUNNER_CLS_MAPPING.get(action)
killcls = RUNNER_CLS_MAPPING.get(runtype)
break break
# Did we find a class that can do it? # Did we find a class that can do it?
if killcls: if killcls:
if killcls in killer_instances: if killcls in killer_instances:
@ -480,7 +467,7 @@ class ProgramRuntime(ComponentBase):
killer_instances[killcls] = killer killer_instances[killcls] = killer
to_kill.append((app_name, killer)) to_kill.append((app_name, killer))
else: else:
msg = "Could not figure out which class to use to stop (%s, %s) of run type (%s)" % (app_name, fn, runtype) msg = "Could not figure out which class to use to stop (%s, %s)" % (app_name, trace_fn)
raise excp.StopException(msg) raise excp.StopException(msg)
return to_kill return to_kill
@ -488,8 +475,8 @@ class ProgramRuntime(ComponentBase):
to_kill = self._locate_killers() to_kill = self._locate_killers()
for (app_name, killer) in to_kill: for (app_name, killer) in to_kill:
killer.stop(app_name) killer.stop(app_name)
LOG.debug("Deleting start trace file [%s]" % (self.starttracereader.trace_fn)) LOG.debug("Deleting start trace file [%s]" % (self.tracereader.filename()))
sh.unlink(self.starttracereader.trace_fn) sh.unlink(self.tracereader.filename())
return len(to_kill) return len(to_kill)
def status(self): def status(self):

12
devstack/components/glance.py
View File

@ -150,7 +150,7 @@ class GlanceInstaller(comp.PythonInstallComponent):
"(and is empty)" % (cache_dir)) "(and is empty)" % (cache_dir))
#destroy then recreate the image cache directory #destroy then recreate the image cache directory
sh.deldir(cache_dir) sh.deldir(cache_dir)
self.tracewriter.make_dir(cache_dir) self.tracewriter.dirs_made(*sh.mkdirslist(cache_dir))
if config.get('default', 'default_store') == 'file': if config.get('default', 'default_store') == 'file':
file_dir = config.get('default', 'filesystem_store_datadir') file_dir = config.get('default', 'filesystem_store_datadir')
if file_dir: if file_dir:
@ -158,26 +158,24 @@ class GlanceInstaller(comp.PythonInstallComponent):
#delete existing images #delete existing images
#and recreate the image directory #and recreate the image directory
sh.deldir(file_dir) sh.deldir(file_dir)
self.tracewriter.make_dir(file_dir) self.tracewriter.dirs_made(*sh.mkdirslist(file_dir))
log_filename = config.get('default', 'log_file') log_filename = config.get('default', 'log_file')
if log_filename: if log_filename:
LOG.info("Ensuring log file %s exists and is empty." % (log_filename)) LOG.info("Ensuring log file %s exists and is empty." % (log_filename))
log_dir = sh.dirname(log_filename) log_dir = sh.dirname(log_filename)
if log_dir: if log_dir:
LOG.info("Ensuring log directory %s exists." % (log_dir)) LOG.info("Ensuring log directory %s exists." % (log_dir))
self.tracewriter.make_dir(log_dir) self.tracewriter.dirs_made(*sh.mkdirslist(log_dir))
#destroy then recreate it (the log file) #destroy then recreate it (the log file)
sh.unlink(log_filename) sh.unlink(log_filename)
sh.touch_file(log_filename) self.tracewriter.file_touched(sh.touch_file(log_filename))
self.tracewriter.file_touched(log_filename)
if config.getboolean('default', 'delayed_delete'): if config.getboolean('default', 'delayed_delete'):
data_dir = config.get('default', 'scrubber_datadir') data_dir = config.get('default', 'scrubber_datadir')
if data_dir: if data_dir:
LOG.info("Ensuring scrubber data dir %s exists and is empty." % (data_dir)) LOG.info("Ensuring scrubber data dir %s exists and is empty." % (data_dir))
#destroy then recreate the scrubber data directory #destroy then recreate the scrubber data directory
sh.deldir(data_dir) sh.deldir(data_dir)
self.tracewriter.make_dir(data_dir) self.tracewriter.dirs_made(*sh.mkdirslist(data_dir))
#we might need to handle more in the future...
#nothing modified so just return the original #nothing modified so just return the original
return contents return contents

13
devstack/components/horizon.py
View File

@ -76,6 +76,9 @@ APACHE_FIXUPS = {
} }
APACHE_FIXUPS_DISTROS = [settings.RHEL6] APACHE_FIXUPS_DISTROS = [settings.RHEL6]
#for when quantum client is not need we need some fake files so python doesn't croak
FAKE_QUANTUM_FILES = ['__init__.py', 'client.py']
#users which apache may not like starting as #users which apache may not like starting as
BAD_APACHE_USERS = ['root'] BAD_APACHE_USERS = ['root']
@ -153,7 +156,7 @@ class HorizonInstaller(comp.PythonInstallComponent):
def _setup_blackhole(self): def _setup_blackhole(self):
#create an empty directory that apache uses as docroot #create an empty directory that apache uses as docroot
self.tracewriter.make_dir(sh.joinpths(self.appdir, BLACKHOLE_DIR)) self.tracewriter.dirs_made(*sh.mkdirslist(sh.joinpths(self.appdir, BLACKHOLE_DIR)))
def _sync_db(self): def _sync_db(self):
#Initialize the horizon database (it stores sessions and notices shown to users). #Initialize the horizon database (it stores sessions and notices shown to users).
@ -175,7 +178,7 @@ class HorizonInstaller(comp.PythonInstallComponent):
def pre_install(self): def pre_install(self):
comp.PythonInstallComponent.pre_install(self) comp.PythonInstallComponent.pre_install(self)
self.tracewriter.make_dir(self.log_dir) self.tracewriter.dirs_made(*sh.mkdirslist(self.log_dir))
def _config_fixups(self): def _config_fixups(self):
#currently just handling rhel fixups #currently just handling rhel fixups
@ -209,9 +212,9 @@ class HorizonInstaller(comp.PythonInstallComponent):
#TODO remove this... #TODO remove this...
quantum_dir = sh.joinpths(self.dash_dir, 'quantum') quantum_dir = sh.joinpths(self.dash_dir, 'quantum')
if not sh.isdir(quantum_dir): if not sh.isdir(quantum_dir):
self.tracewriter.make_dir(quantum_dir) self.tracewriter.dirs_made(*sh.mkdirslist(quantum_dir))
self.tracewriter.touch_file(sh.joinpths(quantum_dir, '__init__.py')) for fn in FAKE_QUANTUM_FILES:
self.tracewriter.touch_file(sh.joinpths(quantum_dir, 'client.py')) self.tracewriter.file_touched(sh.touch_file(sh.joinpths(quantum_dir, fn)))
def post_install(self): def post_install(self):
comp.PythonInstallComponent.post_install(self) comp.PythonInstallComponent.post_install(self)

5
devstack/components/keystone.py
View File

@ -160,11 +160,10 @@ class KeystoneInstaller(comp.PythonInstallComponent):
log_dir = sh.dirname(log_filename) log_dir = sh.dirname(log_filename)
if log_dir: if log_dir:
LOG.info("Ensuring log directory %s exists." % (log_dir)) LOG.info("Ensuring log directory %s exists." % (log_dir))
self.tracewriter.make_dir(log_dir) self.tracewriter.dirs_made(*sh.mkdirslist(log_dir))
#destroy then recreate it (the log file) #destroy then recreate it (the log file)
sh.unlink(log_filename) sh.unlink(log_filename)
sh.touch_file(log_filename) self.tracewriter.file_touched(sh.touch_file(log_filename))
self.tracewriter.file_touched(log_filename)
elif name == CATALOG_CONF: elif name == CATALOG_CONF:
nlines = list() nlines = list()
if utils.service_enabled(settings.SWIFT, self.instances): if utils.service_enabled(settings.SWIFT, self.instances):

20
devstack/components/nova.py
View File

@ -379,13 +379,12 @@ class NovaInstaller(comp.PythonInstallComponent):
def _generate_nova_conf(self): def _generate_nova_conf(self):
LOG.info("Generating dynamic content for nova configuration (%s)." % (API_CONF)) LOG.info("Generating dynamic content for nova configuration (%s)." % (API_CONF))
conf_gen = NovaConfConfigurator(self) conf_gen = NovaConfConfigurator(self)
nova_conf = conf_gen.configure() nova_conf_contents = conf_gen.configure()
tgtfn = self._get_target_config_name(API_CONF) conf_fn = self._get_target_config_name(API_CONF)
LOG.info("Writing nova configuration to %s" % (tgtfn)) LOG.info("Writing nova configuration to %s" % (conf_fn))
LOG.debug(nova_conf) LOG.debug(nova_conf_contents)
self.tracewriter.make_dir(sh.dirname(tgtfn)) self.tracewriter.dirs_made(*sh.mkdirslist(sh.dirname(conf_fn)))
sh.write_file(tgtfn, nova_conf) self.tracewriter.cfg_file_written(sh.write_file(conf_fn, nova_conf_contents))
self.tracewriter.cfg_write(tgtfn)
def _get_source_config(self, config_fn): def _get_source_config(self, config_fn):
name = config_fn name = config_fn
@ -419,11 +418,12 @@ class NovaInstaller(comp.PythonInstallComponent):
# TODO: maybe this should be a subclass that handles these differences # TODO: maybe this should be a subclass that handles these differences
driver_canon = _canon_virt_driver(self.cfg.get('nova', 'virt_driver')) driver_canon = _canon_virt_driver(self.cfg.get('nova', 'virt_driver'))
if (self.distro in POLICY_DISTROS) and driver_canon == virsh.VIRT_TYPE: if (self.distro in POLICY_DISTROS) and driver_canon == virsh.VIRT_TYPE:
dirs_made = list()
with sh.Rooted(True): with sh.Rooted(True):
dirsmade = sh.mkdirslist(sh.dirname(LIBVIRT_POLICY_FN)) dirs_made = sh.mkdirslist(sh.dirname(LIBVIRT_POLICY_FN))
sh.write_file(LIBVIRT_POLICY_FN, LIBVIRT_POLICY_CONTENTS) sh.write_file(LIBVIRT_POLICY_FN, LIBVIRT_POLICY_CONTENTS)
self.tracewriter.dir_made(*dirsmade) self.tracewriter.dirs_made(*dirs_made)
self.tracewriter.cfg_write(LIBVIRT_POLICY_FN) self.tracewriter.cfg_file_written(LIBVIRT_POLICY_FN)
configs_made += 1 configs_made += 1
return configs_made return configs_made

15
devstack/components/swift.py
View File

@ -167,20 +167,23 @@ class SwiftInstaller(comp.PythonInstallComponent):
def _create_nodes(self): def _create_nodes(self):
for i in range(1, 5): for i in range(1, 5):
self.tracewriter.make_dir(sh.joinpths(self.fs_dev, '%d/node' % i)) self.tracewriter.dirs_made(sh.mkdirslist(sh.joinpths(self.fs_dev, '%d/node' % i)))
self.tracewriter.symlink(sh.joinpths(self.fs_dev, str(i)), link_tgt = sh.joinpths(self.datadir, str(i))
sh.joinpths(self.datadir, str(i))) sh.symlink(sh.joinpths(self.fs_dev, str(i)), link_tgt)
self.tracewriter.symlink_made(link_tgt)
start_port = (6010 + (i - 1) * 5) start_port = (6010 + (i - 1) * 5)
self._create_node_config(i, start_port) self._create_node_config(i, start_port)
self._delete_templates() self._delete_templates()
def _turn_on_rsync(self): def _turn_on_rsync(self):
self.tracewriter.symlink(sh.joinpths(self.cfgdir, RSYNC_CONF), RSYNCD_CONF_LOC) sh.symlink(sh.joinpths(self.cfgdir, RSYNC_CONF), RSYNCD_CONF_LOC)
self.tracewriter.symlink_made(RSYNCD_CONF_LOC)
sh.replace_in(RSYNC_CONF_LOC, RSYNC_ON_OFF_RE, 'RSYNC_ENABLE=true', True) sh.replace_in(RSYNC_CONF_LOC, RSYNC_ON_OFF_RE, 'RSYNC_ENABLE=true', True)
def _create_log_dirs(self): def _create_log_dirs(self):
self.tracewriter.make_dir(sh.joinpths(self.logdir, 'hourly')) self.tracewriter.dirs_made(*sh.mkdirslist(sh.joinpths(self.logdir, 'hourly')))
self.tracewriter.symlink(sh.joinpths(self.cfgdir, SYSLOG_CONF), SWIFT_RSYNC_LOC) sh.symlink(sh.joinpths(self.cfgdir, SYSLOG_CONF), SWIFT_RSYNC_LOC)
self.tracewriter.symlink_made(SWIFT_RSYNC_LOC)
def _setup_binaries(self): def _setup_binaries(self):
sh.move(sh.joinpths(self.cfgdir, SWIFT_MAKERINGS), self.makerings_file) sh.move(sh.joinpths(self.cfgdir, SWIFT_MAKERINGS), self.makerings_file)

7
devstack/runners/fork.py
View File

@ -165,11 +165,10 @@ class ForkRunner(base.RunnerBase):
os._exit(0) os._exit(0)
def _do_trace(self, fn, kvs): def _do_trace(self, fn, kvs):
tracefn = tr.touch_trace(self.trace_dir, fn) run_trace = tr.TraceWriter(tr.trace_fn(self.trace_dir, fn))
runtrace = tr.Trace(tracefn)
for (k, v) in kvs.items(): for (k, v) in kvs.items():
runtrace.trace(k, v) run_trace.trace(k, v)
return tracefn return run_trace.filename()
def start(self, app_name, runtime_info): def start(self, app_name, runtime_info):
(program, appdir, program_args) = runtime_info (program, appdir, program_args) = runtime_info

19
devstack/runners/screen.py
View File

@ -78,15 +78,14 @@ class ScreenRunner(base.RunnerBase):
self.socket_dir = sh.joinpths(tempfile.gettempdir(), SCREEN_SOCKET_DIR_NAME) self.socket_dir = sh.joinpths(tempfile.gettempdir(), SCREEN_SOCKET_DIR_NAME)
def stop(self, app_name): def stop(self, app_name):
fn_name = SCREEN_TEMPL % (app_name) trace_fn = tr.trace_fn(self.trace_dir, SCREEN_TEMPL % (app_name))
trace_fn = tr.trace_fn(self.trace_dir, fn_name)
session_id = self._find_session(app_name, trace_fn) session_id = self._find_session(app_name, trace_fn)
self._do_stop(app_name, session_id) self._do_stop(app_name, session_id)
sh.unlink(trace_fn) sh.unlink(trace_fn)
def _find_session(self, app_name, trace_fn): def _find_session(self, app_name, trace_fn):
session_id = None session_id = None
for (key, value) in tr.parse_fn(trace_fn): for (key, value) in tr.TraceReader(trace_fn).read():
if key == SESSION_ID and value: if key == SESSION_ID and value:
session_id = value session_id = value
if not session_id: if not session_id:
@ -200,12 +199,10 @@ class ScreenRunner(base.RunnerBase):
sh.chmod(d, perm) sh.chmod(d, perm)
def _begin_start(self, name, program, args): def _begin_start(self, name, program, args):
fn_name = SCREEN_TEMPL % (name) run_trace = tr.TraceWriter(tr.trace_fn(self.trace_dir, SCREEN_TEMPL % (name)))
tracefn = tr.touch_trace(self.trace_dir, fn_name) run_trace.trace(TYPE, RUN_TYPE)
runtrace = tr.Trace(tracefn) run_trace.trace(NAME, name)
runtrace.trace(TYPE, RUN_TYPE) run_trace.trace(ARGS, json.dumps(args))
runtrace.trace(NAME, name)
runtrace.trace(ARGS, json.dumps(args))
full_cmd = [program] + list(args) full_cmd = [program] + list(args)
session_name = self._get_session() session_name = self._get_session()
inited_screen = False inited_screen = False
@ -216,7 +213,7 @@ class ScreenRunner(base.RunnerBase):
if session_name is None: if session_name is None:
msg = "After initializing screen with session named [%s], no screen session with that name was found!" % (SESSION_NAME) msg = "After initializing screen with session named [%s], no screen session with that name was found!" % (SESSION_NAME)
raise excp.StartException(msg) raise excp.StartException(msg)
runtrace.trace(SESSION_ID, session_name) run_trace.trace(SESSION_ID, session_name)
if inited_screen or not sh.isfile(SCREEN_RC): if inited_screen or not sh.isfile(SCREEN_RC):
rc_gen = ScreenRcGenerator(self) rc_gen = ScreenRcGenerator(self)
rc_contents = rc_gen.create(session_name, self._get_env()) rc_contents = rc_gen.create(session_name, self._get_env())
@ -224,7 +221,7 @@ class ScreenRunner(base.RunnerBase):
LOG.info("Writing your created screen rc file to [%s]" % (out_fn)) LOG.info("Writing your created screen rc file to [%s]" % (out_fn))
sh.write_file(out_fn, rc_contents) sh.write_file(out_fn, rc_contents)
self._do_start(session_name, name, full_cmd) self._do_start(session_name, name, full_cmd)
return tracefn return run_trace.filename()
def start(self, app_name, runtime_info): def start(self, app_name, runtime_info):
(program, _, program_args) = runtime_info (program, _, program_args) = runtime_info

12
devstack/runners/upstart.py
View File

@ -116,12 +116,10 @@ class UpstartRunner(base.RunnerBase):
sh.chmod(cfg_fn, 0666) sh.chmod(cfg_fn, 0666)
def _start(self, app_name, program, program_args): def _start(self, app_name, program, program_args):
fn_name = UPSTART_TEMPL % (app_name) run_trace = tr.TraceWriter(tr.trace_fn(self.trace_dir, UPSTART_TEMPL % (app_name)))
tracefn = tr.touch_trace(self.trace_dir, fn_name) run_trace.trace(TYPE, RUN_TYPE)
runtrace = tr.Trace(tracefn) run_trace.trace(NAME, app_name)
runtrace.trace(TYPE, RUN_TYPE) run_trace.trace(ARGS, json.dumps(program_args))
runtrace.trace(NAME, app_name)
runtrace.trace(ARGS, json.dumps(program_args))
# Emit the start, keep track and only do one per component name # Emit the start, keep track and only do one per component name
component_event = self.component_name + START_EVENT_SUFFIX component_event = self.component_name + START_EVENT_SUFFIX
if component_event in self.events: if component_event in self.events:
@ -131,7 +129,7 @@ class UpstartRunner(base.RunnerBase):
cmd = EMIT_BASE_CMD + [component_event] cmd = EMIT_BASE_CMD + [component_event]
sh.execute(*cmd, run_as_root=True) sh.execute(*cmd, run_as_root=True)
self.events.add(component_event) self.events.add(component_event)
return tracefn return run_trace.filename()
def start(self, app_name, runtime_info): def start(self, app_name, runtime_info):
(program, _, program_args) = runtime_info (program, _, program_args) = runtime_info

3
devstack/shell.py
View File

@ -348,6 +348,7 @@ def append_file(fn, text, flush=True, quiet=False):
f.write(text) f.write(text)
if flush: if flush:
f.flush() f.flush()
return fn
def write_file(fn, text, flush=True, quiet=False): def write_file(fn, text, flush=True, quiet=False):
@ -357,6 +358,7 @@ def write_file(fn, text, flush=True, quiet=False):
f.write(text) f.write(text)
if flush: if flush:
f.flush() f.flush()
return fn
def touch_file(fn, die_if_there=True, quiet=False, file_size=0): def touch_file(fn, die_if_there=True, quiet=False, file_size=0):
@ -369,6 +371,7 @@ def touch_file(fn, die_if_there=True, quiet=False, file_size=0):
if die_if_there: if die_if_there:
msg = "Can not touch & truncate file %s since it already exists" % (fn) msg = "Can not touch & truncate file %s since it already exists" % (fn)
raise excp.FileException(msg) raise excp.FileException(msg)
return fn
def load_file(fn, quiet=False): def load_file(fn, quiet=False):

261
devstack/trace.py
View File

@ -34,7 +34,6 @@ FILE_TOUCHED = "FILE_TOUCHED"
DOWNLOADED = "DOWNLOADED" DOWNLOADED = "DOWNLOADED"
AP_STARTED = "AP_STARTED" AP_STARTED = "AP_STARTED"
PIP_INSTALL = 'PIP_INSTALL' PIP_INSTALL = 'PIP_INSTALL'
EXEC_CMD = 'EXEC_CMD'
#trace file types #trace file types
PY_TRACE = "python" PY_TRACE = "python"
@ -46,153 +45,149 @@ TRACE_VERSION = "TRACE_VERSION"
TRACE_VER = 0x1 TRACE_VER = 0x1
class Trace(object): def trace_fn(root_dir, name):
def __init__(self, tracefn): return sh.joinpths(root_dir, name + TRACE_EXT)
self.tracefn = tracefn
def filename(self):
return self.tracefn class TraceWriter(object):
def __init__(self, trace_filename):
self.trace_fn = trace_filename
self.started = False
def trace(self, cmd, action=None): def trace(self, cmd, action=None):
if action is None: if action is None:
action = date.rcf8222date() action = date.rcf8222date()
line = TRACE_FMT % (cmd, action) if cmd is not None:
sh.append_file(self.tracefn, line) sh.append_file(self.trace_fn, TRACE_FMT % (cmd, action))
def filename(self):
class TraceWriter(object): return self.trace_fn
def __init__(self, root, name):
self.tracer = None
self.root = root
self.name = name
self.filename = None
self.started = False
def _start(self): def _start(self):
if self.started: if self.started:
return return
else: else:
dirs = sh.mkdirslist(self.root) trace_dirs = sh.mkdirslist(sh.dirname(self.trace_fn))
self.filename = touch_trace(self.root, self.name) sh.touch_file(self.trace_fn)
self.tracer = Trace(self.filename) self.trace(TRACE_VERSION, str(TRACE_VER))
self.tracer.trace(TRACE_VERSION, str(TRACE_VER))
if dirs:
for d in dirs:
self.tracer.trace(DIR_MADE, d)
self.started = True self.started = True
self.dirs_made(*trace_dirs)
def py_install(self, name, trace_filename, where): def py_installed(self, name, where):
self._start() self._start()
what = dict() what = dict()
what['name'] = name what['name'] = name
what['trace'] = trace_filename
what['where'] = where what['where'] = where
self.tracer.trace(PYTHON_INSTALL, json.dumps(what)) self.trace(PYTHON_INSTALL, json.dumps(what))
def cfg_write(self, cfgfile): def cfg_file_written(self, fn):
self._start() self._start()
self.tracer.trace(CFG_WRITING_FILE, cfgfile) self.trace(CFG_WRITING_FILE, fn)
def symlink(self, source, link): def symlink_made(self, link):
self._start() self._start()
dirs = sh.symlink(source, link) self.trace(SYMLINK_MAKE, link)
self.dir_made(*dirs)
self.tracer.trace(SYMLINK_MAKE, link)
def downloaded(self, tgt, fromwhere): def download_happened(self, tgt, uri):
self._start() self._start()
what = dict() what = dict()
what['target'] = tgt what['target'] = tgt
what['from'] = fromwhere what['from'] = uri
self.tracer.trace(DOWNLOADED, json.dumps(what)) self.trace(DOWNLOADED, json.dumps(what))
def pip_install(self, name, pip_info): def pip_installed(self, name, pip_info):
self._start() self._start()
what = dict() what = dict()
what['name'] = name what['name'] = name
what['pip_meta'] = pip_info what['pip_meta'] = pip_info
self.tracer.trace(PIP_INSTALL, json.dumps(what)) self.trace(PIP_INSTALL, json.dumps(what))
def make_dir(self, path): def dirs_made(self, *dirs):
self._start()
dirs = sh.mkdirslist(path)
self.dir_made(*dirs)
return path
def touch_file(self, path):
self._start()
sh.touch_file(path)
self.file_touched(path)
return path
def dir_made(self, *dirs):
self._start() self._start()
for d in dirs: for d in dirs:
self.tracer.trace(DIR_MADE, d) self.trace(DIR_MADE, d)
def file_touched(self, fn): def file_touched(self, fn):
self._start() self._start()
self.tracer.trace(FILE_TOUCHED, fn) self.trace(FILE_TOUCHED, fn)
def package_install(self, name, pkg_info): def package_installed(self, name, pkg_info):
self._start() self._start()
what = dict() what = dict()
what['name'] = name what['name'] = name
what['pkg_meta'] = pkg_info what['pkg_meta'] = pkg_info
self.tracer.trace(PKG_INSTALL, json.dumps(what)) self.trace(PKG_INSTALL, json.dumps(what))
def started_info(self, name, info_fn): def started_info(self, name, info_fn):
self._start() self._start()
data = dict() data = dict()
data['name'] = name data['name'] = name
data['trace_fn'] = info_fn data['trace_fn'] = info_fn
self.tracer.trace(AP_STARTED, json.dumps(data)) self.trace(AP_STARTED, json.dumps(data))
def exec_cmd(self, cmd, result):
self._start()
data = dict()
data['cmd'] = cmd
data['result'] = result
self.tracer.trace(EXEC_CMD, json.dumps(data))
class TraceReader(object): class TraceReader(object):
def __init__(self, root, name): def __init__(self, trace_filename):
self.root = root self.trace_fn = trace_filename
self.name = name self.contents = None
self.trace_fn = trace_fn(root, name)
def _readpy(self): def filename(self):
lines = self._read() return self.trace_fn
pyentries = list()
for (cmd, action) in lines:
if cmd == PYTHON_INSTALL and len(action):
jentry = json.loads(action)
if type(jentry) is dict:
pyentries.append(jentry)
return pyentries
def _read(self): def _parse(self):
return parse_name(self.root, self.name) fn = self.trace_fn
if not sh.isfile(fn):
msg = "No trace found at filename %s" % (fn)
raise excp.NoTraceException(msg)
contents = sh.load_file(fn)
lines = contents.splitlines()
accum = list()
for line in lines:
ep = self._split_line(line)
if ep is None:
continue
accum.append(tuple(ep))
return accum
def read(self):
if self.contents is None:
self.contents = self._parse()
return self.contents
def _split_line(self, line):
pieces = line.split("-", 1)
if len(pieces) == 2:
cmd = pieces[0].rstrip()
action = pieces[1].lstrip()
return (cmd, action)
else:
return None
def exists(self): def exists(self):
return sh.exists(self.trace_fn) return sh.exists(self.trace_fn)
def downloaded(self): def py_listing(self):
lines = self._read() lines = self.read()
py_entries = list()
for (cmd, action) in lines:
if cmd == PYTHON_INSTALL and len(action):
entry = json.loads(action)
if type(entry) is dict:
py_entries.append((entry.get("name"), entry.get("where")))
return py_entries
def download_locations(self):
lines = self.read()
locs = list() locs = list()
for (cmd, action) in lines: for (cmd, action) in lines:
if cmd == DOWNLOADED and len(action): if cmd == DOWNLOADED and len(action):
jentry = json.loads(action) entry = json.loads(action)
if type(jentry) is dict: if type(entry) is dict:
locs.append(jentry) locs.append(entry.get('target'))
return locs return locs
def py_listing(self):
return self._readpy()
def files_touched(self): def files_touched(self):
lines = self._read() lines = self.read()
files = list() files = list()
for (cmd, action) in lines: for (cmd, action) in lines:
if cmd == FILE_TOUCHED and len(action): if cmd == FILE_TOUCHED and len(action):
@ -202,7 +197,7 @@ class TraceReader(object):
return files return files
def dirs_made(self): def dirs_made(self):
lines = self._read() lines = self.read()
dirs = list() dirs = list()
for (cmd, action) in lines: for (cmd, action) in lines:
if cmd == DIR_MADE and len(action): if cmd == DIR_MADE and len(action):
@ -214,17 +209,17 @@ class TraceReader(object):
return dirs return dirs
def apps_started(self): def apps_started(self):
lines = self._read() lines = self.read()
files = list() app_info = list()
for (cmd, action) in lines: for (cmd, action) in lines:
if cmd == AP_STARTED and len(action): if cmd == AP_STARTED and len(action):
jdec = json.loads(action) entry = json.loads(action)
if type(jdec) is dict: if type(entry) is dict:
files.append(jdec) app_info.append((entry.get('trace_fn'), entry.get('name')))
return files return app_info
def symlinks_made(self): def symlinks_made(self):
lines = self._read() lines = self.read()
files = list() files = list()
for (cmd, action) in lines: for (cmd, action) in lines:
if cmd == SYMLINK_MAKE and len(action): if cmd == SYMLINK_MAKE and len(action):
@ -235,7 +230,7 @@ class TraceReader(object):
return files return files
def files_configured(self): def files_configured(self):
lines = self._read() lines = self.read()
files = list() files = list()
for (cmd, action) in lines: for (cmd, action) in lines:
if cmd == CFG_WRITING_FILE and len(action): if cmd == CFG_WRITING_FILE and len(action):
@ -245,79 +240,31 @@ class TraceReader(object):
return files return files
def pips_installed(self): def pips_installed(self):
lines = self._read() lines = self.read()
pipsinstalled = dict() pips_installed = dict()
pip_list = list() pip_list = list()
for (cmd, action) in lines: for (cmd, action) in lines:
if cmd == PIP_INSTALL and len(action): if cmd == PIP_INSTALL and len(action):
pip_list.append(action) pip_list.append(action)
for pdata in pip_list: for pip_data in pip_list:
pip_info_full = json.loads(pdata) pip_info_full = json.loads(pip_data)
if type(pip_info_full) is dict: if type(pip_info_full) is dict:
name = pip_info_full.get('name') name = pip_info_full.get('name')
if name and len(name): if name:
pipsinstalled[name] = pip_info_full.get('pip_meta') pips_installed[name] = pip_info_full.get('pip_meta')
return pipsinstalled return pips_installed
def packages_installed(self): def packages_installed(self):
lines = self._read() lines = self.read()
pkgsinstalled = dict() pkgs_installed = dict()
pkg_list = list() pkg_list = list()
for (cmd, action) in lines: for (cmd, action) in lines:
if cmd == PKG_INSTALL and len(action): if cmd == PKG_INSTALL and len(action):
pkg_list.append(action) pkg_list.append(action)
for pdata in pkg_list: for pkg_data in pkg_list:
pkg_info = json.loads(pdata) pkg_info = json.loads(pkg_data)
if type(pkg_info) is dict: if type(pkg_info) is dict:
name = pkg_info.get('name') name = pkg_info.get('name')
if name and len(name): if name:
pkgsinstalled[name] = pkg_info.get('pkg_meta') pkgs_installed[name] = pkg_info.get('pkg_meta')
return pkgsinstalled return pkgs_installed
def trace_fn(rootdir, name):
fullname = name + TRACE_EXT
return sh.joinpths(rootdir, fullname)
def touch_trace(rootdir, name):
tracefn = trace_fn(rootdir, name)
sh.touch_file(tracefn)
return tracefn
def split_line(line):
pieces = line.split("-", 1)
if len(pieces) == 2:
cmd = pieces[0].rstrip()
action = pieces[1].lstrip()
return (cmd, action)
else:
return None
def read(rootdir, name):
pth = trace_fn(rootdir, name)
contents = sh.load_file(pth)
lines = contents.splitlines()
return lines
def parse_fn(fn):
if not sh.isfile(fn):
msg = "No trace found at filename %s" % (fn)
raise excp.NoTraceException(msg)
contents = sh.load_file(fn)
lines = contents.splitlines()
accum = list()
for line in lines:
ep = split_line(line)
if ep is None:
continue
accum.append(tuple(ep))
return accum
def parse_name(rootdir, name):
fn = trace_fn(rootdir, name)
return parse_fn(fn)

30
stack
View File

@ -18,7 +18,6 @@
import logging import logging
import logging.config import logging.config
import os
import sys import sys
import time import time
import traceback import traceback
@ -105,6 +104,23 @@ def run(args):
return True return True
def configure_logging(args):
# Debug by default
root_logger = logging.getLogger('')
root_logger.setLevel(logging.DEBUG)
# Set our pretty logger
console_logger = logging.StreamHandler(sys.stdout)
console_format = '%(levelname)s: @%(name)s : %(message)s'
console_logger.setFormatter(colorlog.TermFormatter(console_format))
root_logger.addHandler(console_logger)
# Adjust logging verbose level based on the command line switch.
log_level = logging.DEBUG if args['verbosity'] >= 2 else logging.INFO
root_logger.setLevel(log_level)
def main(): def main():
#do this first so people can see the help message... #do this first so people can see the help message...
@ -112,16 +128,8 @@ def main():
prog_name = sys.argv[0] prog_name = sys.argv[0]
# Configure logging # Configure logging
root_logger = logging.getLogger('') configure_logging(args)
root_logger.setLevel(logging.DEBUG)
console = logging.StreamHandler(sys.stdout)
console_format = '%(levelname)s: @%(name)s : %(message)s'
console.setFormatter(colorlog.TermFormatter(console_format))
root_logger.addHandler(console)
# Adjust logging verbose level based on the command line switch.
log_level = logging.DEBUG if args['verbosity'] >= 2 else logging.INFO
root_logger.setLevel(log_level)
LOG.debug("Command line options %s" % (args)) LOG.debug("Command line options %s" % (args))
#will need root to setup openstack #will need root to setup openstack