Add custom filters for checking services

These filters can be used to capture a lot of the logic that we
currently have in 'when' statements, about which services are enabled
for a particular host.

In order to use these filters, it is necessary to install the
kolla_ansible python module, and not just the dependencies listed in
requirements.txt. The CI test and quickstart install from source
documentation has been updated accordingly.

Ansible is not currently in OpenStack global requirements, so for unit
tests we avoid a direct dependency on Ansible and provide fakes where
necessary.

Change-Id: Ib91cac3c28e2b5a834c9746b1d2236a309529556

.stestr.conf

@@ -1,4 +1,4 @@
 [DEFAULT]
-test_path=./tests
+test_path=./
 top_dir=./
 

ansible/filter_plugins/services.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2019 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from kolla_ansible import filters
+
+
+class FilterModule(object):
+    """Service filters."""
+
+    def filters(self):
+        return filters.get_filters()

doc/source/user/quickstart.rst

@@ -220,15 +220,15 @@ Install Kolla for development
 
    .. code-block:: console
 
-      pip install -r kolla/requirements.txt
-      pip install -r kolla-ansible/requirements.txt
+      pip install ./kolla
+      pip install ./kolla-ansible
 
    If not using a virtual environment:
 
    .. code-block:: console
 
-      sudo pip install -r kolla/requirements.txt
-      sudo pip install -r kolla-ansible/requirements.txt
+      sudo pip install ./kolla
+      sudo pip install ./kolla-ansible
 
 #. Create the ``/etc/kolla`` directory.
 

kolla_ansible/exception.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2019 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+try:
+    from ansible.errors import AnsibleFilterError
+except ImportError:
+    # NOTE(mgoddard): For unit testing we don't depend on Ansible since it is
+    # not in global requirements.
+    AnsibleFilterError = Exception
+
+
+class FilterError(AnsibleFilterError):
+    """Error during execution of a jinja2 filter."""

kolla_ansible/filters.py

@@ -0,0 +1,107 @@
+# Copyright (c) 2019 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import jinja2
+
+from kolla_ansible import exception
+
+
+def _call_bool_filter(context, value):
+    """Pass a value through the 'bool' filter.
+
+    :param context: Jinja2 Context object.
+    :param value: Value to pass through bool filter.
+    :returns: A boolean.
+    """
+    return context.environment.call_filter("bool", value, context=context)
+
+
+@jinja2.contextfilter
+def service_enabled(context, service):
+    """Return whether a service is enabled.
+
+    :param context: Jinja2 Context object.
+    :param service: Service definition, dict.
+    :returns: A boolean.
+    """
+    enabled = service.get('enabled')
+    if enabled is None:
+        raise exception.FilterError(
+            "Service definition for '%s' does not have an 'enabled' attribute"
+            % service.get("container_name", "<unknown>"))
+    return _call_bool_filter(context, enabled)
+
+
+@jinja2.contextfilter
+def service_mapped_to_host(context, service):
+    """Return whether a service is mapped to this host.
+
+    There are two ways to describe the service to host mapping. The most common
+    is via a 'group' attribute, where the service is mapped to all hosts in the
+    group. The second approach is via a 'host_in_groups' attribute, which is a
+    boolean expression which should be evaluated for every host. The latter
+    approach takes precedence over the first.
+
+    :param context: Jinja2 Context object.
+    :param service: Service definition, dict.
+    :returns: A boolean.
+    """
+    host_in_groups = service.get("host_in_groups")
+    if host_in_groups is not None:
+        return _call_bool_filter(context, host_in_groups)
+
+    group = service.get("group")
+    if group is not None:
+        return group in context.get("groups")
+
+    raise exception.FilterError(
+        "Service definition for '%s' does not have a 'group' or "
+        "'host_in_groups' attribute" %
+        service.get("container_name", "<unknown>"))
+
+
+@jinja2.contextfilter
+def service_enabled_and_mapped_to_host(context, service):
+    """Return whether a service is enabled and mapped to this host.
+
+    :param context: Jinja2 Context object.
+    :param service: Service definition, dict.
+    :returns: A boolean.
+    """
+    return (service_enabled(context, service) and
+            service_mapped_to_host(context, service))
+
+
+@jinja2.contextfilter
+def select_services_enabled_and_mapped_to_host(context, services):
+    """Select services that are enabled and mapped to this host.
+
+    :param context: Jinja2 Context object.
+    :param services: Service definitions, dict.
+    :returns: A dict containing enabled services mapped to this host.
+    """
+    return {service_name: service
+            for service_name, service in services.items()
+            if service_enabled_and_mapped_to_host(context, service)}
+
+
+def get_filters():
+    return {
+        "service_enabled": service_enabled,
+        "service_mapped_to_host": service_mapped_to_host,
+        "service_enabled_and_mapped_to_host": (
+            service_enabled_and_mapped_to_host),
+        "select_services_enabled_and_mapped_to_host": (
+            select_services_enabled_and_mapped_to_host),
+    }

kolla_ansible/tests/unit/test_filters.py

@@ -0,0 +1,173 @@
+# Copyright (c) 2019 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+import jinja2
+import mock
+
+from kolla_ansible import exception
+from kolla_ansible import filters
+
+
+def _to_bool(value):
+    """Simplified version of the bool filter.
+
+    Avoids having a dependency on Ansible in unit tests.
+    """
+    if value == 'yes':
+        return True
+    if value == 'no':
+        return False
+    return bool(value)
+
+
+class TestFilters(unittest.TestCase):
+
+    def setUp(self):
+        # Bandit complains about Jinja2 autoescaping without nosec.
+        self.env = jinja2.Environment()  # nosec
+        self.env.filters['bool'] = _to_bool
+        self.context = self._make_context()
+
+    def _make_context(self, parent=None):
+        if parent is None:
+            parent = {}
+        return self.env.context_class(
+            self.env, parent=parent, name='dummy', blocks={})
+
+    def test_service_enabled_true(self):
+        service = {
+            'enabled': True
+        }
+        self.assertTrue(filters.service_enabled(self.context, service))
+
+    def test_service_enabled_yes(self):
+        service = {
+            'enabled': 'yes'
+        }
+        self.assertTrue(filters.service_enabled(self.context, service))
+
+    def test_service_enabled_false(self):
+        service = {
+            'enabled': False
+        }
+        self.assertFalse(filters.service_enabled(self.context, service))
+
+    def test_service_enabled_no(self):
+        service = {
+            'enabled': 'no'
+        }
+        self.assertFalse(filters.service_enabled(self.context, service))
+
+    def test_service_enabled_no_attr(self):
+        service = {}
+        self.assertRaises(exception.FilterError,
+                          filters.service_enabled, self.context, service)
+
+    def test_service_mapped_to_host_host_in_groups_true(self):
+        service = {
+            'host_in_groups': True
+        }
+        self.assertTrue(filters.service_mapped_to_host(self.context, service))
+
+    def test_service_mapped_to_host_host_in_groups_yes(self):
+        service = {
+            'host_in_groups': 'yes'
+        }
+        self.assertTrue(filters.service_mapped_to_host(self.context, service))
+
+    def test_service_mapped_to_host_host_in_groups_false(self):
+        service = {
+            'host_in_groups': False
+        }
+        self.assertFalse(filters.service_mapped_to_host(self.context, service))
+
+    def test_service_mapped_to_host_host_in_groups_no(self):
+        service = {
+            'host_in_groups': 'no'
+        }
+        self.assertFalse(filters.service_mapped_to_host(self.context, service))
+
+    def test_service_mapped_to_host_in_group(self):
+        service = {
+            'group': 'foo'
+        }
+        context = self._make_context({'groups': ['foo', 'bar']})
+        self.assertTrue(filters.service_mapped_to_host(context, service))
+
+    def test_service_mapped_to_host_not_in_group(self):
+        service = {
+            'group': 'foo'
+        }
+        context = self._make_context({'groups': ['bar']})
+        self.assertFalse(filters.service_mapped_to_host(context, service))
+
+    def test_service_mapped_to_host_no_attr(self):
+        service = {}
+        self.assertRaises(exception.FilterError,
+                          filters.service_mapped_to_host, self.context,
+                          service)
+
+    @mock.patch.object(filters, 'service_enabled')
+    @mock.patch.object(filters, 'service_mapped_to_host')
+    def test_service_enabled_and_mapped_to_host(self,  mock_mapped,
+                                                mock_enabled):
+        service = {}
+        mock_enabled.return_value = True
+        mock_mapped.return_value = True
+        self.assertTrue(filters.service_enabled_and_mapped_to_host(
+            self.context, service))
+        mock_enabled.assert_called_once_with(self.context, service)
+        mock_mapped.assert_called_once_with(self.context, service)
+
+    @mock.patch.object(filters, 'service_enabled')
+    @mock.patch.object(filters, 'service_mapped_to_host')
+    def test_service_enabled_and_mapped_to_host_disabled(self,  mock_mapped,
+                                                         mock_enabled):
+        service = {}
+        mock_enabled.return_value = False
+        mock_mapped.return_value = True
+        self.assertFalse(filters.service_enabled_and_mapped_to_host(
+            self.context, service))
+        mock_enabled.assert_called_once_with(self.context, service)
+        self.assertFalse(mock_mapped.called)
+
+    @mock.patch.object(filters, 'service_enabled')
+    @mock.patch.object(filters, 'service_mapped_to_host')
+    def test_service_enabled_and_mapped_to_host_not_mapped(self,  mock_mapped,
+                                                           mock_enabled):
+        service = {}
+        mock_enabled.return_value = True
+        mock_mapped.return_value = False
+        self.assertFalse(filters.service_enabled_and_mapped_to_host(
+            self.context, service))
+        mock_enabled.assert_called_once_with(self.context, service)
+        mock_mapped.assert_called_once_with(self.context, service)
+
+    @mock.patch.object(filters, 'service_enabled_and_mapped_to_host')
+    def test_select_services_enabled_and_mapped_to_host(self, mock_seamth):
+        services = {
+            'foo': object(),
+            'bar': object(),
+            'baz': object(),
+        }
+        mock_seamth.side_effect = lambda _, s: s != services['bar']
+        result = filters.select_services_enabled_and_mapped_to_host(
+            self.context, services)
+        expected = {
+            'foo': services['foo'],
+            'baz': services['baz'],
+        }
+        self.assertEqual(expected, result)

releasenotes/notes/jinja-filters-818d5bb97ddc75c6.yaml

@@ -0,0 +1,13 @@
+---
+upgrade:
+  - |
+    When installing ``kolla-ansible`` from source, the ``kolla_ansible`` python
+    module must now be installed in addition to the python dependencies listed
+    in ``requirements.txt``. This is done via::
+
+        pip install /path/to/kolla-ansible
+
+    If the git repository is in the current directory, use the following
+    to avoid installing the package from PyPI::
+
+        pip install ./kolla-ansible

tests/run.yml

@@ -133,9 +133,9 @@
               dest: ironic-agent.kernel
       when: scenario == "ironic"
 
-    - name: install kolla-ansible requirements
+    - name: install kolla-ansible
       pip:
-        requirements: "{{ kolla_ansible_src_dir }}/requirements.txt"
+        name: "{{ kolla_ansible_src_dir }}"
       become: true
 
     - name: copy passwords.yml file
@@ -311,9 +311,9 @@
               when: "{{ is_ceph }}"
           when: item.when | default(true)
 
-        - name: upgrade kolla-ansible requirements
+        - name: upgrade kolla-ansible
           pip:
-            requirements: "{{ kolla_ansible_src_dir }}/requirements.txt"
+            name: "{{ kolla_ansible_src_dir }}"
           become: true
 
         # Update passwords.yml to include any new passwords added in this