Automatically set LIBS_FROM_GIT based on required projects
If a project shows up in zuul's required-projects list, add it to LIBS_FROM_GIT automatically. This way, when a user specifies that a job requires a zuul-project, it gets used in testing, but otherwise, it doesn't (pypi is used instead). Also add information about what happens behind the scenes for both LIBS_FROM_GIT and plugin dependencies. This moves the check performed in check_libs_from_git to a helper function which is installed for most kinds of installations. This means that if someone sets LIBS_FROM_GIT to "foobar", devstack won't error anymore, as nothing is going to try to install foobar, therefore the check won't run on that. However, as we move to automated generation of the local config, that error is not likely to happen. This check was originally added due to an error in the upper-constraints file (where a constraint name did not match a package name). This location of the check would still catch that type of error. Change-Id: Ifcf3ad008cf42d3d4762cfb3b6c31c93cfeb40db
This commit is contained in:
James E. Blair
parent
6f27fca4a7
commit
e1edde38ed
23
inc/python
23
inc/python
@ -435,22 +435,6 @@ function lib_installed_from_git {
|
||||
[[ -n $(pip list --format=columns 2>/dev/null | awk "/^$safe_name/ {print \$3}") ]]
|
||||
}
|
||||
|
||||
# check that everything that's in LIBS_FROM_GIT was actually installed
|
||||
# correctly, this helps double check issues with library fat fingering.
|
||||
function check_libs_from_git {
|
||||
local lib=""
|
||||
local not_installed=""
|
||||
for lib in $(echo ${LIBS_FROM_GIT} | tr "," " "); do
|
||||
if ! lib_installed_from_git "$lib"; then
|
||||
not_installed+=" $lib"
|
||||
fi
|
||||
done
|
||||
# if anything is not installed, say what it is.
|
||||
if [[ -n "$not_installed" ]]; then
|
||||
die $LINENO "The following LIBS_FROM_GIT were not installed correct: $not_installed"
|
||||
fi
|
||||
}
|
||||
|
||||
# setup a library by name. If we are trying to use the library from
|
||||
# git, we'll do a git based install, otherwise we'll punt and the
|
||||
# library should be installed by a requirements pull from another
|
||||
@ -561,6 +545,13 @@ function _setup_package_with_constraints_edit {
|
||||
|
||||
setup_package $project_dir "$flags" $extras
|
||||
|
||||
# If this project is in LIBS_FROM_GIT, verify it was actually installed
|
||||
# correctly. This helps catch errors caused by constraints mismatches.
|
||||
if use_library_from_git "$project_dir"; then
|
||||
if ! lib_installed_from_git "$project_dir"; then
|
||||
die $LINENO "The following LIBS_FROM_GIT was not installed correctly: $project_dir"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# ``pip install -e`` the package, which processes the dependencies
|
||||
|
||||
@ -20,6 +20,14 @@ Write the local.conf file for use by devstack
|
||||
bash shell variables, and will be ordered so that variables used by
|
||||
later entries appear first.
|
||||
|
||||
As a special case, the variable ``LIBS_FROM_GIT`` will be
|
||||
constructed automatically from the projects which appear in the
|
||||
``required-projects`` list defined by the job. To instruct
|
||||
devstack to install a library from source rather than pypi, simply
|
||||
add that library to the job's ``required-projects`` list. To
|
||||
override the automatically-generated value, set ``LIBS_FROM_GIT``
|
||||
in ``devstack_localrc`` to the desired value.
|
||||
|
||||
.. zuul:rolevar:: devstack_local_conf
|
||||
:type: dict
|
||||
|
||||
@ -75,3 +83,7 @@ Write the local.conf file for use by devstack
|
||||
A dictionary mapping a plugin name to a git repo location. If the
|
||||
location is a non-empty string, then an ``enable_plugin`` line will
|
||||
be emmitted for the plugin name.
|
||||
|
||||
If a plugin declares a dependency on another plugin (via
|
||||
``plugin_requires`` in the plugin's settings file), this role will
|
||||
automatically emit ``enable_plugin`` lines in the correct order.
|
||||
|
||||
@ -207,17 +207,17 @@ class PluginGraph(DependencyGraph):
|
||||
class LocalConf(object):
|
||||
|
||||
def __init__(self, localrc, localconf, base_services, services, plugins,
|
||||
base_dir):
|
||||
base_dir, projects):
|
||||
self.localrc = []
|
||||
self.meta_sections = {}
|
||||
self.plugin_deps = {}
|
||||
self.base_dir = base_dir
|
||||
self.projects = projects
|
||||
if plugins:
|
||||
self.handle_plugins(plugins)
|
||||
if services or base_services:
|
||||
self.handle_services(base_services, services or {})
|
||||
if localrc:
|
||||
self.handle_localrc(localrc)
|
||||
self.handle_localrc(localrc)
|
||||
if localconf:
|
||||
self.handle_localconf(localconf)
|
||||
|
||||
@ -241,9 +241,22 @@ class LocalConf(object):
|
||||
self.localrc.append('enable_service {}'.format(k))
|
||||
|
||||
def handle_localrc(self, localrc):
|
||||
vg = VarGraph(localrc)
|
||||
for k, v in vg.getVars():
|
||||
self.localrc.append('{}={}'.format(k, v))
|
||||
lfg = False
|
||||
if localrc:
|
||||
vg = VarGraph(localrc)
|
||||
for k, v in vg.getVars():
|
||||
self.localrc.append('{}={}'.format(k, v))
|
||||
if k == 'LIBS_FROM_GIT':
|
||||
lfg = True
|
||||
|
||||
if not lfg and self.projects:
|
||||
required_projects = []
|
||||
for project_name, project_info in self.projects.items():
|
||||
if project_info.get('required'):
|
||||
required_projects.append(project_info['short_name'])
|
||||
if required_projects:
|
||||
self.localrc.append('LIBS_FROM_GIT={}'.format(
|
||||
','.join(required_projects)))
|
||||
|
||||
def handle_localconf(self, localconf):
|
||||
for phase, phase_data in localconf.items():
|
||||
@ -277,6 +290,7 @@ def main():
|
||||
local_conf=dict(type='dict'),
|
||||
base_dir=dict(type='path'),
|
||||
path=dict(type='str'),
|
||||
projects=dict(type='dict'),
|
||||
)
|
||||
)
|
||||
|
||||
@ -286,7 +300,8 @@ def main():
|
||||
p.get('base_services'),
|
||||
p.get('services'),
|
||||
p.get('plugins'),
|
||||
p.get('base_dir'))
|
||||
p.get('base_dir'),
|
||||
p.get('projects'))
|
||||
lc.write(p['path'])
|
||||
|
||||
module.exit_json()
|
||||
|
||||
@ -56,7 +56,8 @@ class TestDevstackLocalConf(unittest.TestCase):
|
||||
p.get('base_services'),
|
||||
p.get('services'),
|
||||
p.get('plugins'),
|
||||
p.get('base_dir'))
|
||||
p.get('base_dir'),
|
||||
p.get('projects'))
|
||||
lc.write(p['path'])
|
||||
|
||||
plugins = []
|
||||
@ -66,6 +67,7 @@ class TestDevstackLocalConf(unittest.TestCase):
|
||||
plugins.append(line.split()[1])
|
||||
self.assertEqual(['bar', 'baz', 'foo'], plugins)
|
||||
|
||||
|
||||
def test_plugin_deps(self):
|
||||
"Test that plugins with dependencies work"
|
||||
os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', 'devstack'))
|
||||
@ -101,20 +103,80 @@ class TestDevstackLocalConf(unittest.TestCase):
|
||||
plugins=plugins,
|
||||
base_dir=self.tmpdir,
|
||||
path=os.path.join(self.tmpdir, 'test.local.conf'))
|
||||
|
||||
def test_libs_from_git(self):
|
||||
"Test that LIBS_FROM_GIT is auto-generated"
|
||||
projects = {
|
||||
'git.openstack.org/openstack/nova': {
|
||||
'required': True,
|
||||
'short_name': 'nova',
|
||||
},
|
||||
'git.openstack.org/openstack/oslo.messaging': {
|
||||
'required': True,
|
||||
'short_name': 'oslo.messaging',
|
||||
},
|
||||
'git.openstack.org/openstack/devstack-plugin': {
|
||||
'required': False,
|
||||
'short_name': 'devstack-plugin',
|
||||
},
|
||||
}
|
||||
p = dict(base_services=[],
|
||||
base_dir='./test',
|
||||
path=os.path.join(self.tmpdir, 'test.local.conf'),
|
||||
projects=projects)
|
||||
lc = LocalConf(p.get('localrc'),
|
||||
p.get('local_conf'),
|
||||
p.get('base_services'),
|
||||
p.get('services'),
|
||||
p.get('plugins'),
|
||||
p.get('base_dir'))
|
||||
p.get('base_dir'),
|
||||
p.get('projects'))
|
||||
lc.write(p['path'])
|
||||
|
||||
plugins = []
|
||||
lfg = None
|
||||
with open(p['path']) as f:
|
||||
for line in f:
|
||||
if line.startswith('enable_plugin'):
|
||||
plugins.append(line.split()[1])
|
||||
self.assertEqual(['foo', 'bar'], plugins)
|
||||
if line.startswith('LIBS_FROM_GIT'):
|
||||
lfg = line.strip().split('=')[1]
|
||||
self.assertEqual('nova,oslo.messaging', lfg)
|
||||
|
||||
def test_overridelibs_from_git(self):
|
||||
"Test that LIBS_FROM_GIT can be overridden"
|
||||
localrc = {'LIBS_FROM_GIT': 'oslo.db'}
|
||||
projects = {
|
||||
'git.openstack.org/openstack/nova': {
|
||||
'required': True,
|
||||
'short_name': 'nova',
|
||||
},
|
||||
'git.openstack.org/openstack/oslo.messaging': {
|
||||
'required': True,
|
||||
'short_name': 'oslo.messaging',
|
||||
},
|
||||
'git.openstack.org/openstack/devstack-plugin': {
|
||||
'required': False,
|
||||
'short_name': 'devstack-plugin',
|
||||
},
|
||||
}
|
||||
p = dict(localrc=localrc,
|
||||
base_services=[],
|
||||
base_dir='./test',
|
||||
path=os.path.join(self.tmpdir, 'test.local.conf'),
|
||||
projects=projects)
|
||||
lc = LocalConf(p.get('localrc'),
|
||||
p.get('local_conf'),
|
||||
p.get('base_services'),
|
||||
p.get('services'),
|
||||
p.get('plugins'),
|
||||
p.get('base_dir'),
|
||||
p.get('projects'))
|
||||
lc.write(p['path'])
|
||||
|
||||
lfg = None
|
||||
with open(p['path']) as f:
|
||||
for line in f:
|
||||
if line.startswith('LIBS_FROM_GIT'):
|
||||
lfg = line.strip().split('=')[1]
|
||||
self.assertEqual('oslo.db', lfg)
|
||||
|
||||
def test_plugin_circular_deps(self):
|
||||
"Test that plugins with circular dependencies fail"
|
||||
|
||||
@ -9,3 +9,4 @@
|
||||
localrc: "{{ devstack_localrc|default(omit) }}"
|
||||
local_conf: "{{ devstack_local_conf|default(omit) }}"
|
||||
base_dir: "{{ devstack_base_dir|default(omit) }}"
|
||||
projects: "{{ zuul.projects }}"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user