diff --git a/inc/python b/inc/python index e074ea498f..37b4617777 100644 --- a/inc/python +++ b/inc/python @@ -435,22 +435,6 @@ function lib_installed_from_git { [[ -n $(pip list --format=columns 2>/dev/null | awk "/^$safe_name/ {print \$3}") ]] } -# check that everything that's in LIBS_FROM_GIT was actually installed -# correctly, this helps double check issues with library fat fingering. -function check_libs_from_git { - local lib="" - local not_installed="" - for lib in $(echo ${LIBS_FROM_GIT} | tr "," " "); do - if ! lib_installed_from_git "$lib"; then - not_installed+=" $lib" - fi - done - # if anything is not installed, say what it is. - if [[ -n "$not_installed" ]]; then - die $LINENO "The following LIBS_FROM_GIT were not installed correct: $not_installed" - fi -} - # setup a library by name. If we are trying to use the library from # git, we'll do a git based install, otherwise we'll punt and the # library should be installed by a requirements pull from another @@ -561,6 +545,13 @@ function _setup_package_with_constraints_edit { setup_package $project_dir "$flags" $extras + # If this project is in LIBS_FROM_GIT, verify it was actually installed + # correctly. This helps catch errors caused by constraints mismatches. + if use_library_from_git "$project_dir"; then + if ! lib_installed_from_git "$project_dir"; then + die $LINENO "The following LIBS_FROM_GIT was not installed correctly: $project_dir" + fi + fi } # ``pip install -e`` the package, which processes the dependencies diff --git a/roles/write-devstack-local-conf/README.rst b/roles/write-devstack-local-conf/README.rst index 73f9f0d6fd..bfce9c98cd 100644 --- a/roles/write-devstack-local-conf/README.rst +++ b/roles/write-devstack-local-conf/README.rst @@ -20,6 +20,14 @@ Write the local.conf file for use by devstack bash shell variables, and will be ordered so that variables used by later entries appear first. + As a special case, the variable ``LIBS_FROM_GIT`` will be + constructed automatically from the projects which appear in the + ``required-projects`` list defined by the job. To instruct + devstack to install a library from source rather than pypi, simply + add that library to the job's ``required-projects`` list. To + override the automatically-generated value, set ``LIBS_FROM_GIT`` + in ``devstack_localrc`` to the desired value. + .. zuul:rolevar:: devstack_local_conf :type: dict @@ -75,3 +83,7 @@ Write the local.conf file for use by devstack A dictionary mapping a plugin name to a git repo location. If the location is a non-empty string, then an ``enable_plugin`` line will be emmitted for the plugin name. + + If a plugin declares a dependency on another plugin (via + ``plugin_requires`` in the plugin's settings file), this role will + automatically emit ``enable_plugin`` lines in the correct order. diff --git a/roles/write-devstack-local-conf/library/devstack_local_conf.py b/roles/write-devstack-local-conf/library/devstack_local_conf.py index 746f54f921..9728fef37b 100644 --- a/roles/write-devstack-local-conf/library/devstack_local_conf.py +++ b/roles/write-devstack-local-conf/library/devstack_local_conf.py @@ -207,17 +207,17 @@ class PluginGraph(DependencyGraph): class LocalConf(object): def __init__(self, localrc, localconf, base_services, services, plugins, - base_dir): + base_dir, projects): self.localrc = [] self.meta_sections = {} self.plugin_deps = {} self.base_dir = base_dir + self.projects = projects if plugins: self.handle_plugins(plugins) if services or base_services: self.handle_services(base_services, services or {}) - if localrc: - self.handle_localrc(localrc) + self.handle_localrc(localrc) if localconf: self.handle_localconf(localconf) @@ -241,9 +241,22 @@ class LocalConf(object): self.localrc.append('enable_service {}'.format(k)) def handle_localrc(self, localrc): - vg = VarGraph(localrc) - for k, v in vg.getVars(): - self.localrc.append('{}={}'.format(k, v)) + lfg = False + if localrc: + vg = VarGraph(localrc) + for k, v in vg.getVars(): + self.localrc.append('{}={}'.format(k, v)) + if k == 'LIBS_FROM_GIT': + lfg = True + + if not lfg and self.projects: + required_projects = [] + for project_name, project_info in self.projects.items(): + if project_info.get('required'): + required_projects.append(project_info['short_name']) + if required_projects: + self.localrc.append('LIBS_FROM_GIT={}'.format( + ','.join(required_projects))) def handle_localconf(self, localconf): for phase, phase_data in localconf.items(): @@ -277,6 +290,7 @@ def main(): local_conf=dict(type='dict'), base_dir=dict(type='path'), path=dict(type='str'), + projects=dict(type='dict'), ) ) @@ -286,7 +300,8 @@ def main(): p.get('base_services'), p.get('services'), p.get('plugins'), - p.get('base_dir')) + p.get('base_dir'), + p.get('projects')) lc.write(p['path']) module.exit_json() diff --git a/roles/write-devstack-local-conf/library/test.py b/roles/write-devstack-local-conf/library/test.py index 843ca6e9fd..7ccb68f08d 100644 --- a/roles/write-devstack-local-conf/library/test.py +++ b/roles/write-devstack-local-conf/library/test.py @@ -56,7 +56,8 @@ class TestDevstackLocalConf(unittest.TestCase): p.get('base_services'), p.get('services'), p.get('plugins'), - p.get('base_dir')) + p.get('base_dir'), + p.get('projects')) lc.write(p['path']) plugins = [] @@ -66,6 +67,7 @@ class TestDevstackLocalConf(unittest.TestCase): plugins.append(line.split()[1]) self.assertEqual(['bar', 'baz', 'foo'], plugins) + def test_plugin_deps(self): "Test that plugins with dependencies work" os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', 'devstack')) @@ -101,20 +103,80 @@ class TestDevstackLocalConf(unittest.TestCase): plugins=plugins, base_dir=self.tmpdir, path=os.path.join(self.tmpdir, 'test.local.conf')) + + def test_libs_from_git(self): + "Test that LIBS_FROM_GIT is auto-generated" + projects = { + 'git.openstack.org/openstack/nova': { + 'required': True, + 'short_name': 'nova', + }, + 'git.openstack.org/openstack/oslo.messaging': { + 'required': True, + 'short_name': 'oslo.messaging', + }, + 'git.openstack.org/openstack/devstack-plugin': { + 'required': False, + 'short_name': 'devstack-plugin', + }, + } + p = dict(base_services=[], + base_dir='./test', + path=os.path.join(self.tmpdir, 'test.local.conf'), + projects=projects) lc = LocalConf(p.get('localrc'), p.get('local_conf'), p.get('base_services'), p.get('services'), p.get('plugins'), - p.get('base_dir')) + p.get('base_dir'), + p.get('projects')) lc.write(p['path']) - plugins = [] + lfg = None with open(p['path']) as f: for line in f: - if line.startswith('enable_plugin'): - plugins.append(line.split()[1]) - self.assertEqual(['foo', 'bar'], plugins) + if line.startswith('LIBS_FROM_GIT'): + lfg = line.strip().split('=')[1] + self.assertEqual('nova,oslo.messaging', lfg) + + def test_overridelibs_from_git(self): + "Test that LIBS_FROM_GIT can be overridden" + localrc = {'LIBS_FROM_GIT': 'oslo.db'} + projects = { + 'git.openstack.org/openstack/nova': { + 'required': True, + 'short_name': 'nova', + }, + 'git.openstack.org/openstack/oslo.messaging': { + 'required': True, + 'short_name': 'oslo.messaging', + }, + 'git.openstack.org/openstack/devstack-plugin': { + 'required': False, + 'short_name': 'devstack-plugin', + }, + } + p = dict(localrc=localrc, + base_services=[], + base_dir='./test', + path=os.path.join(self.tmpdir, 'test.local.conf'), + projects=projects) + lc = LocalConf(p.get('localrc'), + p.get('local_conf'), + p.get('base_services'), + p.get('services'), + p.get('plugins'), + p.get('base_dir'), + p.get('projects')) + lc.write(p['path']) + + lfg = None + with open(p['path']) as f: + for line in f: + if line.startswith('LIBS_FROM_GIT'): + lfg = line.strip().split('=')[1] + self.assertEqual('oslo.db', lfg) def test_plugin_circular_deps(self): "Test that plugins with circular dependencies fail" diff --git a/roles/write-devstack-local-conf/tasks/main.yaml b/roles/write-devstack-local-conf/tasks/main.yaml index 2a9f8985fc..a294cae608 100644 --- a/roles/write-devstack-local-conf/tasks/main.yaml +++ b/roles/write-devstack-local-conf/tasks/main.yaml @@ -9,3 +9,4 @@ localrc: "{{ devstack_localrc|default(omit) }}" local_conf: "{{ devstack_local_conf|default(omit) }}" base_dir: "{{ devstack_base_dir|default(omit) }}" + projects: "{{ zuul.projects }}" diff --git a/stack.sh b/stack.sh index 9b496c0e20..5643b4db1b 100755 --- a/stack.sh +++ b/stack.sh @@ -1402,11 +1402,6 @@ fi # Check the status of running services service_check -# ensure that all the libraries we think we installed from git, -# actually were. -check_libs_from_git - - # Configure nova cellsv2 # ----------------------