Zuul: support plugin dependencies

Change-Id: I81302e8988fe6498fea9f08ed66f5d0cc1fce161

.gitignore

@@ -3,6 +3,7 @@
 *.log
 *.log.[1-9]
 *.pem
+*.pyc
 .localrc.auto
 .localrc.password
 .prereqs

roles/write-devstack-local-conf/library/devstack_local_conf.py

@@ -14,16 +14,69 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import re
 
 
-class VarGraph(object):
+class DependencyGraph(object):
     # This is based on the JobGraph from Zuul.
 
+    def __init__(self):
+        self._names = set()
+        self._dependencies = {}  # dependent_name -> set(parent_names)
+
+    def add(self, name, dependencies):
+        # Append the dependency information
+        self._dependencies.setdefault(name, set())
+        try:
+            for dependency in dependencies:
+                # Make sure a circular dependency is never created
+                ancestors = self._getParentNamesRecursively(
+                    dependency, soft=True)
+                ancestors.add(dependency)
+                if name in ancestors:
+                    raise Exception("Dependency cycle detected in {}".
+                                    format(name))
+                self._dependencies[name].add(dependency)
+        except Exception:
+            del self._dependencies[name]
+            raise
+
+    def getDependenciesRecursively(self, parent):
+        dependencies = []
+
+        current_dependencies = self._dependencies[parent]
+        for current in current_dependencies:
+            if current not in dependencies:
+                dependencies.append(current)
+            for dep in self.getDependenciesRecursively(current):
+                if dep not in dependencies:
+                    dependencies.append(dep)
+        return dependencies
+
+    def _getParentNamesRecursively(self, dependent, soft=False):
+        all_parent_items = set()
+        items_to_iterate = set([dependent])
+        while len(items_to_iterate) > 0:
+            current_item = items_to_iterate.pop()
+            current_parent_items = self._dependencies.get(current_item)
+            if current_parent_items is None:
+                if soft:
+                    current_parent_items = set()
+                else:
+                    raise Exception("Dependent item {} not found: ".format(
+                                    dependent))
+            new_parent_items = current_parent_items - all_parent_items
+            items_to_iterate |= new_parent_items
+            all_parent_items |= new_parent_items
+        return all_parent_items
+
+
+class VarGraph(DependencyGraph):
     def __init__(self, vars):
+        super(VarGraph, self).__init__()
         self.vars = {}
         self._varnames = set()
-        self._dependencies = {}  # dependent_var_name -> set(parent_var_names)
         for k, v in vars.items():
             self._varnames.add(k)
         for k, v in vars.items():
@@ -38,28 +91,21 @@ class VarGraph(object):
             raise Exception("Variable {} already added".format(key))
         self.vars[key] = value
         # Append the dependency information
-        self._dependencies.setdefault(key, set())
+        dependencies = set()
+        for dependency in self.getDependencies(value):
+            if dependency == key:
+                # A variable is allowed to reference itself; no
+                # dependency link needed in that case.
+                continue
+            if dependency not in self._varnames:
+                # It's not necessary to create a link for an
+                # external variable.
+                continue
+            dependencies.add(dependency)
         try:
-            for dependency in self.getDependencies(value):
+            self.add(key, dependencies)
-                if dependency == key:
-                    # A variable is allowed to reference itself; no
-                    # dependency link needed in that case.
-                    continue
-                if dependency not in self._varnames:
-                    # It's not necessary to create a link for an
-                    # external variable.
-                    continue
-                # Make sure a circular dependency is never created
-                ancestor_vars = self._getParentVarNamesRecursively(
-                    dependency, soft=True)
-                ancestor_vars.add(dependency)
-                if any((key == anc_var) for anc_var in ancestor_vars):
-                    raise Exception("Dependency cycle detected in var {}".
-                                    format(key))
-                self._dependencies[key].add(dependency)
         except Exception:
             del self.vars[key]
-            del self._dependencies[key]
             raise
 
     def getVars(self):
@@ -67,48 +113,105 @@ class VarGraph(object):
         keys = sorted(self.vars.keys())
         seen = set()
         for key in keys:
-            dependencies = self.getDependentVarsRecursively(key)
+            dependencies = self.getDependenciesRecursively(key)
             for var in dependencies + [key]:
                 if var not in seen:
                     ret.append((var, self.vars[var]))
                     seen.add(var)
         return ret
 
-    def getDependentVarsRecursively(self, parent_var):
-        dependent_vars = []
 
-        current_dependent_vars = self._dependencies[parent_var]
+class PluginGraph(DependencyGraph):
-        for current_var in current_dependent_vars:
+    def __init__(self, base_dir, plugins):
-            if current_var not in dependent_vars:
+        super(PluginGraph, self).__init__()
-                dependent_vars.append(current_var)
+        # The dependency trees expressed by all the plugins we found
-            for dep in self.getDependentVarsRecursively(current_var):
+        # (which may be more than those the job is using).
-                if dep not in dependent_vars:
+        self._plugin_dependencies = {}
-                    dependent_vars.append(dep)
+        self.loadPluginNames(base_dir)
-        return dependent_vars
 
-    def _getParentVarNamesRecursively(self, dependent_var, soft=False):
+        self.plugins = {}
-        all_parent_vars = set()
+        self._pluginnames = set()
-        vars_to_iterate = set([dependent_var])
+        for k, v in plugins.items():
-        while len(vars_to_iterate) > 0:
+            self._pluginnames.add(k)
-            current_var = vars_to_iterate.pop()
+        for k, v in plugins.items():
-            current_parent_vars = self._dependencies.get(current_var)
+            self._addPlugin(k, str(v))
-            if current_parent_vars is None:
+
-                if soft:
+    def loadPluginNames(self, base_dir):
-                    current_parent_vars = set()
+        if base_dir is None:
-                else:
+            return
-                    raise Exception("Dependent var {} not found: ".format(
+        git_roots = []
-                                    dependent_var))
+        for root, dirs, files in os.walk(base_dir):
-            new_parent_vars = current_parent_vars - all_parent_vars
+            if '.git' not in dirs:
-            vars_to_iterate |= new_parent_vars
+                continue
-            all_parent_vars |= new_parent_vars
+            # Don't go deeper than git roots
-        return all_parent_vars
+            dirs[:] = []
+            git_roots.append(root)
+        for root in git_roots:
+            devstack = os.path.join(root, 'devstack')
+            if not (os.path.exists(devstack) and os.path.isdir(devstack)):
+                continue
+            settings = os.path.join(devstack, 'settings')
+            if not (os.path.exists(settings) and os.path.isfile(settings)):
+                continue
+            self.loadDevstackPluginInfo(settings)
+
+    define_re = re.compile(r'^define_plugin\s+(\w+).*')
+    require_re = re.compile(r'^plugin_requires\s+(\w+)\s+(\w+).*')
+    def loadDevstackPluginInfo(self, fn):
+        name = None
+        reqs = set()
+        with open(fn) as f:
+            for line in f:
+                m = self.define_re.match(line)
+                if m:
+                    name = m.group(1)
+                m = self.require_re.match(line)
+                if m:
+                    if name == m.group(1):
+                        reqs.add(m.group(2))
+        if name and reqs:
+            self._plugin_dependencies[name] = reqs
+
+    def getDependencies(self, value):
+        return self._plugin_dependencies.get(value, [])
+
+    def _addPlugin(self, key, value):
+        if key in self.plugins:
+            raise Exception("Plugin {} already added".format(key))
+        self.plugins[key] = value
+        # Append the dependency information
+        dependencies = set()
+        for dependency in self.getDependencies(key):
+            if dependency == key:
+                continue
+            dependencies.add(dependency)
+        try:
+            self.add(key, dependencies)
+        except Exception:
+            del self.plugins[key]
+            raise
+
+    def getPlugins(self):
+        ret = []
+        keys = sorted(self.plugins.keys())
+        seen = set()
+        for key in keys:
+            dependencies = self.getDependenciesRecursively(key)
+            for plugin in dependencies + [key]:
+                if plugin not in seen:
+                    ret.append((plugin, self.plugins[plugin]))
+                    seen.add(plugin)
+        return ret
 
 
 class LocalConf(object):
 
-    def __init__(self, localrc, localconf, base_services, services, plugins):
+    def __init__(self, localrc, localconf, base_services, services, plugins,
+                 base_dir):
         self.localrc = []
         self.meta_sections = {}
+        self.plugin_deps = {}
+        self.base_dir = base_dir
         if plugins:
             self.handle_plugins(plugins)
         if services or base_services:
@@ -119,7 +222,8 @@ class LocalConf(object):
             self.handle_localconf(localconf)
 
     def handle_plugins(self, plugins):
-        for k, v in plugins.items():
+        pg = PluginGraph(self.base_dir, plugins)
+        for k, v in pg.getPlugins():
             if v:
                 self.localrc.append('enable_plugin {} {}'.format(k, v))
 
@@ -171,6 +275,7 @@ def main():
             services=dict(type='dict'),
             localrc=dict(type='dict'),
             local_conf=dict(type='dict'),
+            base_dir=dict(type='path'),
             path=dict(type='str'),
         )
     )
@@ -180,14 +285,18 @@ def main():
                    p.get('local_conf'),
                    p.get('base_services'),
                    p.get('services'),
-                   p.get('plugins'))
+                   p.get('plugins'),
+                   p.get('base_dir'))
     lc.write(p['path'])
 
     module.exit_json()
 
 
-from ansible.module_utils.basic import *  # noqa
+try:
-from ansible.module_utils.basic import AnsibleModule
+    from ansible.module_utils.basic import *  # noqa
+    from ansible.module_utils.basic import AnsibleModule
+except ImportError:
+    pass
 
 if __name__ == '__main__':
     main()

roles/write-devstack-local-conf/library/test.py

@@ -0,0 +1,166 @@
+# Copyright (C) 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import tempfile
+import unittest
+
+from devstack_local_conf import LocalConf
+from collections import OrderedDict
+
+class TestDevstackLocalConf(unittest.TestCase):
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
+
+    def test_plugins(self):
+        "Test that plugins without dependencies work"
+        localrc = {'test_localrc': '1'}
+        local_conf = {'install':
+                      {'nova.conf':
+                       {'main':
+                        {'test_conf': '2'}}}}
+        services = {'cinder': True}
+        # We use ordereddict here to make sure the plugins are in the
+        # *wrong* order for testing.
+        plugins = OrderedDict([
+            ('bar', 'git://git.openstack.org/openstack/bar-plugin'),
+            ('foo', 'git://git.openstack.org/openstack/foo-plugin'),
+            ('baz', 'git://git.openstack.org/openstack/baz-plugin'),
+            ])
+        p = dict(localrc=localrc,
+                 local_conf=local_conf,
+                 base_services=[],
+                 services=services,
+                 plugins=plugins,
+                 base_dir='./test',
+                 path=os.path.join(self.tmpdir, 'test.local.conf'))
+        lc = LocalConf(p.get('localrc'),
+                       p.get('local_conf'),
+                       p.get('base_services'),
+                       p.get('services'),
+                       p.get('plugins'),
+                       p.get('base_dir'))
+        lc.write(p['path'])
+
+        plugins = []
+        with open(p['path']) as f:
+            for line in f:
+                if line.startswith('enable_plugin'):
+                    plugins.append(line.split()[1])
+        self.assertEqual(['bar', 'baz', 'foo'], plugins)
+
+    def test_plugin_deps(self):
+        "Test that plugins with dependencies work"
+        os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', 'devstack'))
+        os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', '.git'))
+        os.makedirs(os.path.join(self.tmpdir, 'bar-plugin', 'devstack'))
+        os.makedirs(os.path.join(self.tmpdir, 'bar-plugin', '.git'))
+        with open(os.path.join(
+                self.tmpdir,
+                'foo-plugin', 'devstack', 'settings'), 'w') as f:
+            f.write('define_plugin foo\n')
+        with open(os.path.join(
+                self.tmpdir,
+                'bar-plugin', 'devstack', 'settings'), 'w') as f:
+            f.write('define_plugin bar\n')
+            f.write('plugin_requires bar foo\n')
+
+        localrc = {'test_localrc': '1'}
+        local_conf = {'install':
+                      {'nova.conf':
+                       {'main':
+                        {'test_conf': '2'}}}}
+        services = {'cinder': True}
+        # We use ordereddict here to make sure the plugins are in the
+        # *wrong* order for testing.
+        plugins = OrderedDict([
+            ('bar', 'git://git.openstack.org/openstack/bar-plugin'),
+            ('foo', 'git://git.openstack.org/openstack/foo-plugin'),
+            ])
+        p = dict(localrc=localrc,
+                 local_conf=local_conf,
+                 base_services=[],
+                 services=services,
+                 plugins=plugins,
+                 base_dir=self.tmpdir,
+                 path=os.path.join(self.tmpdir, 'test.local.conf'))
+        lc = LocalConf(p.get('localrc'),
+                       p.get('local_conf'),
+                       p.get('base_services'),
+                       p.get('services'),
+                       p.get('plugins'),
+                       p.get('base_dir'))
+        lc.write(p['path'])
+
+        plugins = []
+        with open(p['path']) as f:
+            for line in f:
+                if line.startswith('enable_plugin'):
+                    plugins.append(line.split()[1])
+        self.assertEqual(['foo', 'bar'], plugins)
+
+    def test_plugin_circular_deps(self):
+        "Test that plugins with circular dependencies fail"
+        os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', 'devstack'))
+        os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', '.git'))
+        os.makedirs(os.path.join(self.tmpdir, 'bar-plugin', 'devstack'))
+        os.makedirs(os.path.join(self.tmpdir, 'bar-plugin', '.git'))
+        with open(os.path.join(
+                self.tmpdir,
+                'foo-plugin', 'devstack', 'settings'), 'w') as f:
+            f.write('define_plugin foo\n')
+            f.write('plugin_requires foo bar\n')
+        with open(os.path.join(
+                self.tmpdir,
+                'bar-plugin', 'devstack', 'settings'), 'w') as f:
+            f.write('define_plugin bar\n')
+            f.write('plugin_requires bar foo\n')
+
+        localrc = {'test_localrc': '1'}
+        local_conf = {'install':
+                      {'nova.conf':
+                       {'main':
+                        {'test_conf': '2'}}}}
+        services = {'cinder': True}
+        # We use ordereddict here to make sure the plugins are in the
+        # *wrong* order for testing.
+        plugins = OrderedDict([
+            ('bar', 'git://git.openstack.org/openstack/bar-plugin'),
+            ('foo', 'git://git.openstack.org/openstack/foo-plugin'),
+            ])
+        p = dict(localrc=localrc,
+                 local_conf=local_conf,
+                 base_services=[],
+                 services=services,
+                 plugins=plugins,
+                 base_dir=self.tmpdir,
+                 path=os.path.join(self.tmpdir, 'test.local.conf'))
+        with self.assertRaises(Exception):
+            lc = LocalConf(p.get('localrc'),
+                           p.get('local_conf'),
+                           p.get('base_services'),
+                           p.get('services'),
+                           p.get('plugins'),
+                           p.get('base_dir'))
+            lc.write(p['path'])
+
+
+if __name__ == '__main__':
+    unittest.main()

roles/write-devstack-local-conf/tasks/main.yaml

@@ -8,3 +8,4 @@
     services: "{{ devstack_services|default(omit) }}"
     localrc: "{{ devstack_localrc|default(omit) }}"
     local_conf: "{{ devstack_local_conf|default(omit) }}"
+    base_dir: "{{ devstack_base_dir|default(omit) }}"

tests/test_write_devstack_local_conf_role.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+TOP=$(cd $(dirname "$0")/.. && pwd)
+
+# Import common functions
+source $TOP/functions
+source $TOP/tests/unittest.sh
+
+python ./roles/write-devstack-local-conf/library/test.py