x/anvil
Code Issues Proposed changes

Updated so that we can have more than 1 python app setup by an installer (ie for horizon...)

Browse Source
This commit is contained in:
Joshua Harlow 2012-01-18 17:24:19 -08:00
parent feb0a25d34
commit 63c0d88c16
9 changed files with 226 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
conf/pkgs/general.json
View File

@ -87,6 +87,11 @@
"allowed": ">=",
"removable" : false
},
"python-coverage": {
"version": "3.4-1",
"allowed": ">=",
"removable": true
},
"euca2ools": {
"version": "2.0.0~bzr464-0ubuntu2",
"allowed": ">=",

214
conf/pkgs/horizon.json
View File

@ -1,108 +1,116 @@
# This is a extended json package definition file
# We allow simple comments (lines starting with a hash symbol)
{
"ubuntu-oneiric": {
"apache2": {
"version": "2.2.20-1ubuntu1",
"allowed": ">="
"ubuntu-oneiric": {
"apache2": {
"version": "2.2.20-1ubuntu1.1",
"allowed": ">=",
"removable": true
},
"libapache2-mod-wsgi": {
"version": "3.3-2ubuntu3",
"allowed": ">=",
"removable": true
},
"python-dateutil": {
"version": "1.4.1-4",
"allowed": ">="
},
"python-paste": {
"version": "1.7.5.1-4ubuntu1",
"allowed": ">=",
"removable": true
},
"python-pastedeploy": {
"version": "1.5.0-2",
"allowed": ">=",
"removable": true
},
"python-anyjson": {
"version": "0.3.1-1",
"allowed": ">=",
"removable": true
},
"python-routes": {
"version": "1.12.3-1",
"allowed": ">=",
"removable": true
},
"python-xattr": {
"version": "0.6-1ubuntu2",
"allowed": ">=",
"removable": true
},
"python-sqlalchemy": {
"version": "0.6.8-1",
"allowed": ">=",
"removable": true
},
"python-webob": {
"version": "1.0.8-1",
"allowed": ">=",
"removable": true
},
"python-kombu": {
"version": "1.0.4-2",
"allowed": ">=",
"removable": true
},
"python-eventlet": {
"version": "0.9.15-0ubuntu4",
"allowed": ">=",
"removable": true
},
"python-nose": {
"version": "1.0.0-1ubuntu1",
"allowed": ">=",
"removable": true
},
"python-sphinx": {
"version": "1.0.7+dfsg-1",
"allowed": ">=",
"removable": true
},
"python-mox": {
"version": "0.5.3-1ubuntu4",
"allowed": ">=",
"removable": true
},
"python-cherrypy3": {
"version": "3.1.2-1",
"allowed": ">=",
"removable": true
},
"python-django": {
"version": "1.3-2ubuntu1",
"allowed": ">=",
"removable": true
},
"python-django-mailer": {
"version": "0.2a1.dev3-0ubuntu1",
"allowed": ">=",
"removable": true
},
"python-django-nose": {
"version": "0.1.2-2",
"allowed": ">=",
"removable": true
},
"python-django-registration": {
"version": "0.7-2",
"allowed": ">=",
"removable": true
},
"python-cloudfiles": {
"version": "1.7.9.2-0ubuntu1",
"allowed": ">=",
"removable": true
},
"python-migrate": {
"version": "0.7.1-1",
"allowed": ">=",
"removable": true
}
},
"libapache2-mod-wsgi": {
"version": "3.3-2ubuntu3",
"allowed": ">="
},
"python-dateutil": {
"version": "1.4.1-4",
"allowed": ">="
},
"python-paste": {
"version": "1.7.5.1-4ubuntu1",
"allowed": ">="
},
"python-pastedeploy": {
"version": "1.5.0-2",
"allowed": ">="
},
"python-anyjson": {
"version": "0.3.1-1",
"allowed": ">="
},
"python-routes": {
"version": "1.12.3-1",
"allowed": ">="
},
"python-xattr": {
"version": "0.6-1ubuntu2",
"allowed": ">="
},
"python-sqlalchemy": {
"version": "0.6.8-1",
"allowed": ">="
},
"python-webob": {
"version": "1.0.8-1",
"allowed": ">="
},
"python-kombu": {
"version": "1.0.4-2",
"allowed": ">="
},
"pylint": {
"version": "0.23.0-1",
"allowed": ">="
},
"pep8": {
"version": "0.6.1-2ubuntu1",
"allowed": ">="
},
"python-eventlet": {
"version": "0.9.15-0ubuntu4",
"allowed": ">="
},
"python-nose": {
"version": "1.0.0-1ubuntu1",
"allowed": ">="
},
"python-sphinx": {
"version": "1.0.7+dfsg-1",
"allowed": ">="
},
"python-mox": {
"version": "0.5.3-1ubuntu4",
"allowed": ">="
},
"python-coverage": {
"version": "3.4-1",
"allowed": ">="
},
"python-cherrypy3": {
"version": "3.1.2-1",
"allowed": ">="
},
"python-django": {
"version": "1.3-2ubuntu1",
"allowed": ">="
},
"python-django-mailer": {
"version": "0.2a1.dev3-0ubuntu1",
"allowed": ">="
},
"python-django-nose": {
"version": "0.1.2-2",
"allowed": ">="
},
"python-django-registration": {
"version": "0.7-2",
"allowed": ">="
},
"python-cloudfiles": {
"version": "1.7.9.2-0ubuntu1",
"allowed": ">="
},
"python-migrate": {
"version": "0.7.1-1",
"allowed": ">="
}
},
"rhel-6": {
}
"rhel-6": {}
}

85
devstack/Component.py
View File

@ -13,6 +13,9 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
import os.path
#TODO fix these
from Util import (component_pths,
get_pkg_list,
@ -161,8 +164,14 @@ class PkgInstallComponent(ComponentBase, InstallComponent):
def _get_config_files(self):
return list()
def _config_adjust(contents, name):
def _config_adjust(self, contents, name):
return contents
def _get_full_config_name(self, name):
return joinpths(self.cfgdir, name)
def _get_source_config_name(self, name):
return joinpths(STACK_CONFIG_DIR, self.component_name, name)
def configure(self):
dirsmade = mkdirslist(self.cfgdir)
@ -170,9 +179,14 @@ class PkgInstallComponent(ComponentBase, InstallComponent):
configs = self._get_config_files()
am = len(configs)
for fn in configs:
#get the params and where it should come from and where it should go
parameters = self._get_param_map(fn)
sourcefn = joinpths(STACK_CONFIG_DIR, self.component_name, fn)
tgtfn = joinpths(self.cfgdir, fn)
sourcefn = self._get_source_config_name(fn)
tgtfn = self._get_full_config_name(fn)
#ensure directory is there (if not created previously)
dirsmade = mkdirslist(os.path.dirname(tgtfn))
self.tracewriter.dir_made(*dirsmade)
#now configure it
LOG.info("Configuring template file %s" % (sourcefn))
contents = load_file(sourcefn)
LOG.info("Replacing parameters in file %s" % (sourcefn))
@ -181,8 +195,8 @@ class PkgInstallComponent(ComponentBase, InstallComponent):
LOG.debug("Applying side-effects of param replacement for template %s" % (sourcefn))
contents = self._config_adjust(contents, fn)
LOG.info("Writing configuration file %s" % (tgtfn))
write_file(tgtfn, contents)
#this trace is used to remove the files configured
write_file(tgtfn, contents)
self.tracewriter.cfg_write(tgtfn)
return am
@ -191,6 +205,14 @@ class PythonInstallComponent(PkgInstallComponent):
def __init__(self, component_name, *args, **kargs):
PkgInstallComponent.__init__(self, component_name, *args, **kargs)
def _get_python_directories(self):
pylist = list()
pylist.append({
'name': 'base',
'work_dir': self.appdir,
})
return pylist
def _python_install(self):
pips = get_pip_list(self.distro, self.component_name)
#install any need pip items
@ -198,13 +220,25 @@ class PythonInstallComponent(PkgInstallComponent):
Pip.install(pips)
for name in pips.keys():
self.tracewriter.pip_install(name, pips.get(name))
#do the actual python install
dirsmade = mkdirslist(self.tracedir)
self.tracewriter.dir_made(*dirsmade)
recordwhere = touch_trace(self.tracedir, PY_TRACE)
self.tracewriter.py_install(recordwhere)
(sysout, stderr) = execute(*PY_INSTALL, cwd=self.appdir, run_as_root=True)
write_file(recordwhere, sysout)
#now setup python
pydirs = self._get_python_directories()
if(len(pydirs)):
LOG.info("Setting up %s python directories" % (len(pydirs)))
dirsmade = mkdirslist(self.tracedir)
self.tracewriter.dir_made(*dirsmade)
for pydir_info in pydirs:
name = pydir_info.get("name")
working_dir = pydir_info.get('work_dir', self.appdir)
py_trace_name = "%s-%s" % (PY_TRACE, name)
recordwhere = touch_trace(self.tracedir, py_trace_name)
self.tracewriter.file_touched(recordwhere)
(sysout, stderr) = execute(*PY_INSTALL, cwd=working_dir, run_as_root=True)
combined_output = "===STDOUT===" + os.linesep
combined_output += sysout + os.linesep
combined_output += "===STDERR===" + os.linesep
combined_output += stderr + os.linesep
write_file(recordwhere, combined_output)
self.tracewriter.py_install(name, recordwhere)
# Overridden
def install(self):
@ -272,7 +306,10 @@ class PythonUninstallComponent(PkgUninstallComponent):
def _uninstall_python(self):
pylisting = self.tracereader.py_listing()
if(pylisting and len(pylisting)):
execute(*PY_UNINSTALL, cwd=self.appdir, run_as_root=True)
LOG.info("Uninstalling %s python setups" % (len(pylisting)))
for entry in pylisting:
where = entry.get('where')
execute(*PY_UNINSTALL, cwd=where, run_as_root=True)
class ProgramRuntime(ComponentBase, RuntimeComponent):
@ -313,7 +350,7 @@ class ProgramRuntime(ComponentBase, RuntimeComponent):
return False
def _get_apps_to_start(self):
raise NotImplementedError()
return list()
def _get_app_options(self, app):
return list()
@ -335,25 +372,29 @@ class ProgramRuntime(ComponentBase, RuntimeComponent):
#this fns list will have info about what was started
fns = list()
apps = self._get_apps_to_start()
for app in apps:
for app_info in apps:
#extract needed keys
app_name = app_info.get("name")
app_pth = app_info.get("path", app_name)
app_dir = app_info.get("app_dir", self.appdir)
#adjust the program options now that we have real locations
params = self._get_param_map(app)
program_opts = self._get_app_options(app)
params = self._get_param_map(app_name)
program_opts = self._get_app_options(app_name)
if(params and program_opts):
adjusted_opts = list()
for opt in program_opts:
adjusted_opts.append(param_replace(opt, params))
program_opts = adjusted_opts
LOG.info("Starting %s with options [%s]" % (app, ", ".join(program_opts)))
LOG.info("Starting [%s] with options [%s]" % (app_name, ", ".join(program_opts)))
#start it with the given settings
fn = starter.start(app, app, *program_opts, app_dir=self.appdir, trace_dir=self.tracedir)
if(fn and len(fn)):
fn = starter.start(app_name, app_pth, *program_opts, app_dir=app_dir, trace_dir=self.tracedir)
if(fn):
fns.append(fn)
LOG.info("Started %s, details are in %s" % (app, fn))
LOG.info("Started %s, details are in %s" % (app_name, fn))
#this trace is used to locate details about what to stop
self.tracewriter.started_info(app, fn)
self.tracewriter.started_info(app_name, fn)
else:
LOG.info("Started %s" % (app))
LOG.info("Started %s" % (app_name))
return fns
def stop(self):

9
devstack/Glance.py
View File

@ -46,6 +46,7 @@ APP_OPTIONS = {
'glance-registry': ['--config-file', joinpths('%ROOT%', "etc", REG_CONF)]
}
CONFIG_ACTUAL_DIR = 'etc'
BIN_DIR = 'bin'
class GlanceUninstaller(PythonUninstallComponent):
@ -60,7 +61,13 @@ class GlanceRuntime(PythonRuntime):
self.cfgdir = joinpths(self.appdir, CONFIG_ACTUAL_DIR)
def _get_apps_to_start(self):
return sorted(APP_OPTIONS.keys())
apps = list()
for app_name in APP_OPTIONS.keys():
apps.append({
'name': app_name,
'path': joinpths(self.appdir, BIN_DIR, app_name),
})
return apps
def _get_app_options(self, app):
return APP_OPTIONS.get(app)

9
devstack/Horizon.py
View File

@ -34,7 +34,16 @@ class HorizonUninstaller(PythonUninstallComponent):
class HorizonInstaller(PythonInstallComponent):
def __init__(self, *args, **kargs):
PythonInstallComponent.__init__(self, TYPE, *args, **kargs)
self.gitloc = self.cfg.get("git", "horizon_repo")
self.brch = self.cfg.get("git", "horizon_branch")
def _get_download_locations(self):
places = PythonInstallComponent._get_download_locations(self)
places.append({
'uri': self.gitloc,
'branch': self.brch,
})
return places
class HorizonRuntime(NullRuntime):
def __init__(self, *args, **kargs):

8
devstack/Keystone.py
View File

@ -136,7 +136,13 @@ class KeystoneRuntime(PythonRuntime):
self.bindir = joinpths(self.appdir, BIN_DIR)
def _get_apps_to_start(self):
return sorted(APP_OPTIONS.keys())
apps = list()
for app_name in APP_OPTIONS.keys():
apps.append({
'name': app_name,
'path': joinpths(self.bindir, app_name),
})
return apps
def _get_app_options(self, app):
return APP_OPTIONS.get(app)

20
devstack/Trace.py
View File

@ -80,9 +80,12 @@ class TraceWriter():
self.tracer.trace(DIR_MADE, d)
self.started = True
def py_install(self, where):
def py_install(self, name, where):
self._start()
self.tracer.trace(PYTHON_INSTALL, where)
what = dict()
what['name'] = name
what['where'] = where
self.tracer.trace(PYTHON_INSTALL, json.dumps(what))
def cfg_write(self, cfgfile):
self._start()
@ -134,16 +137,13 @@ class TraceReader():
def _readpy(self):
lines = self._read()
pyfn = None
pylines = list()
pyentries = list()
for (cmd, action) in lines:
if(cmd == PYTHON_INSTALL and len(action)):
pyfn = action
break
if(pyfn != None):
lines = load_file(pyfn).splitlines()
pylines = lines
return pylines
jentry = json.loads(action)
if(type(jentry) is dict):
pyentries.append(jentry)
return pyentries
def _read(self):
return parse_name(self.root, self.name)

21
devstack/runners/Foreground.py
View File

@ -46,6 +46,7 @@ PID_FN = "PID_FN"
STDOUT_FN = "STDOUT_FN"
STDERR_FN = "STDERR_FN"
NAME = "NAME"
FORK_TEMPL = "%s.fork"
class ForegroundRunner(Runner.Runner):
@ -54,10 +55,11 @@ class ForegroundRunner(Runner.Runner):
def stop(self, name, *args, **kargs):
rootdir = kargs.get("trace_dir")
pidfile = joinpths(rootdir, name + ".pid")
stderr = joinpths(rootdir, name + ".stderr")
stdout = joinpths(rootdir, name + ".stdout")
tfname = Trace.trace_fn(rootdir, name)
fn_name = FORK_TEMPL % (name)
pidfile = joinpths(rootdir, fn_name + ".pid")
stderr = joinpths(rootdir, fn_name + ".stderr")
stdout = joinpths(rootdir, fn_name + ".stdout")
tfname = Trace.trace_fn(rootdir, fn_name)
if(isfile(pidfile) and isfile(tfname)):
pid = int(load_file(pidfile).strip())
killed = False
@ -99,16 +101,17 @@ class ForegroundRunner(Runner.Runner):
def start(self, name, program, *args, **kargs):
tracedir = kargs.get("trace_dir")
appdir = kargs.get("app_dir")
pidfile = joinpths(tracedir, name + ".pid")
stderr = joinpths(tracedir, name + ".stderr")
stdout = joinpths(tracedir, name + ".stdout")
tracefn = Trace.trace_fn(tracedir, name)
tracefn = Trace.touch_trace(tracedir, name)
fn_name = FORK_TEMPL % (name)
pidfile = joinpths(tracedir, fn_name + ".pid")
stderr = joinpths(tracedir, fn_name + ".stderr")
stdout = joinpths(tracedir, fn_name + ".stdout")
tracefn = Trace.touch_trace(tracedir, fn_name)
runtrace = Trace.Trace(tracefn)
runtrace.trace(RUN, RUN_TYPE)
runtrace.trace(PID_FN, pidfile)
runtrace.trace(STDERR_FN, stderr)
runtrace.trace(STDOUT_FN, stdout)
LOG.info("Forking [%s] by running command [%s]" % (name, program))
#fork to get daemon out
pid = os.fork()
if(pid == 0):

2
stack
View File

@ -173,7 +173,7 @@ def runner(action_name, component_set, distro, root_dir, program_args):
LOG.info("Performed %s downloads." % (str(am_downloaded)))
LOG.info("Configuring %s." % (c))
am_configured = instance.configure()
LOG.info("Configuring %s files." % (str(am_configured)))
LOG.info("Configured %s files." % (str(am_configured)))
LOG.info("Pre-installing %s." % (c))
instance.pre_install()
LOG.info("Installing %s." % (c))