Add a custom filter for parsing instance metadata

TripleO's TLS implementation using nova in the undercloud relies on
instance metadata that contains the services a particular instance is
going to host. For example, metadata for a controller (e.g.,
controller-0) will contain JSON that describes the services and
interfaces that are supposed to be hosted on controller (e.g.,
haproxy/controller-0.ctlplane.example.test).

For TLS, we need to generate certificates for those services
automatically. This requires us to parse the metadata for the instance,
determine the services that are supposed to the hosted by that
instance, and then curate unique principals that we add to FreeIPA.
Once the appropriate hosts (e.g., controller-0.example.test), sub-hosts
(e.g., controller-0.ctlplane.example.test), and principals (e.g.,
haproxy/controller-0.ctlplane.example.test) have been added to FreeIPA,
we can generate certificates for that service and configure it to use
TLS.

The commit attempts to introduce a filter that parses metadata from the
instance and returns a list of tuples to the caller. Each tuple is
unique to each service being hosted on an instance and we use them to
generate the requisite entities in FreeIPA for TLS certificates using
ansible-freeipa later.

The functionality was formally a part of novajoin, which is a nova
metadata service.

This patch adds unit tests and relevant env configuration.

Change-Id: Ibbac29a9d2570280cad0f426ad0b0367587bfee7

.pre-commit-config.yaml

@@ -0,0 +1,49 @@
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.1.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: mixed-line-ending
+      - id: check-byte-order-marker
+      - id: check-executables-have-shebangs
+      - id: check-merge-conflict
+      - id: debug-statements
+      - id: flake8
+        entry: flake8 --ignore=E24,E121,E122,E123,E124,E126,E226,E265,E305,E402,F401,F405,E501,E704,F403,F841,W503
+        # TODO(cloudnull): These codes were added to pass the lint check.
+        #                  All of these ignore codes should be resolved in
+        #                  future PRs.
+      - id: check-yaml
+        files: .*\.(yaml|yml)$
+  - repo: https://github.com/adrienverge/yamllint.git
+    rev: v1.15.0
+    hooks:
+      - id: yamllint
+        files: \.(yaml|yml)$
+        types: [file, yaml]
+        entry: yamllint --strict -f parsable
+  - repo: https://github.com/ansible/ansible-lint
+    rev: v4.1.1a2
+    hooks:
+      - id: ansible-lint
+        files: \.(yaml|yml)$
+        entry: >-
+          ansible-lint --force-color -v -x "ANSIBLE0006,ANSIBLE0007,ANSIBLE0010,ANSIBLE0012,ANSIBLE0013,ANSIBLE0016"
+          --exclude=tripleo_ansible/roles.galaxy
+        # TODO(cloudnull): These codes were added to pass the lint check.
+        #                  Things found within roles.galaxy are external
+        #                  and not something maintained here.
+  - repo: https://github.com/openstack-dev/bashate.git
+    rev: 0.6.0
+    hooks:
+      - id: bashate
+        entry: bashate --error . --verbose --ignore=E006,E040
+        # Run bashate check for all bash scripts
+        # Ignores the following rules:
+        # E006: Line longer than 79 columns (as many scripts use jinja
+        #       templating, this is very difficult)
+        # E040: Syntax error determined using `bash -n` (as many scripts
+        #       use jinja templating, this will often fail and the syntax
+        #       error will be discovered in execution anyway)

.stestr.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+test_path=${TEST_PATH:-./tripleo_ipa/tests/}
+top_dir=./

ansible-requirements.txt

@@ -0,0 +1 @@
+ansible>=2.8

setup.cfg

@@ -1,5 +1,5 @@
 [metadata]
-name = tripleo_ipa
+name = tripleo-ipa
 summary = Ansible assets for interacting with FreeIPA on behalf of TripleO
 description-file =
     README.rst
@@ -8,9 +8,25 @@ author = RedHat
 home-page = https://opendev.org/x/tripleo-ipa
 classifier =
     License :: OSI Approved :: Apache Software License
-    Development Status :: 2 - Pre-Alpha
+    Development Status :: 4 - Beta
     Intended Audience :: Developers
     Intended Audience :: System Administrators
     Intended Audience :: Information Technology
     Topic :: Utilities
 
+[global]
+setup-hooks =
+    pbr.hooks.setup_hook
+
+[files]
+data_files =
+    share/ansible/tripleo-playbooks/ = tripleo_ipa/playbooks/*
+    share/ansible/plugins/ = tripleo_ipa/ansible_plugins/*
+    share/ansible/roles/ = tripleo_ipa/roles/*
+
+[wheel]
+universal = 1
+
+[pbr]
+skip_authors = True
+skip_changelog = True

test-requirements.txt

@@ -1,2 +1,9 @@
-ansible-lint
-yamllint
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+ansible-lint # MIT
+pre-commit # MIT
+mock>=2.0.0 # BSD
+stestr>=2.0.0 # Apache-2.0
+oslotest>=3.2.0 # Apache-2.0

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 minversion = 2.0
 # add docs to the list of environments once we actually have docs to generate
-envlist = molecule, linters
+envlist = py36,pep8,molecule,linters
 skipdist = True
 
 [testenv]
@@ -9,12 +9,18 @@ usedevelop = True
 install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt} {opts} {packages}
 passenv = *
 sitepackages = True
+deps =
+   -r {toxinidir}/ansible-requirements.txt
+   -r {toxinidir}/test-requirements.txt
+commands = stestr run {posargs}
+whitelist_externals =
+  tox
+
+[testenv:molecule]
 deps =
    -r {toxinidir}/molecule-requirements.txt
 changedir = {toxinidir}/tripleo_ipa
 commands = molecule test --all
-whitelist_externals =
-  tox
 
 [testenv:ansible-lint]
 deps = {[testenv:linters]deps}
@@ -32,3 +38,8 @@ deps =
 commands =
   {[testenv:ansible-lint]commands}
   {[testenv:yamllint]commands}
+
+[testenv:pep8]
+envdir = {toxworkdir}/linters
+commands =
+    python -m pre_commit run flake8 -a

tripleo_ipa/ansible_plugins/filter/service_metadata.py

@@ -0,0 +1,82 @@
+#!/usr/bin/python
+#
+# Copyright 2020 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def parse_service_metadata(service_metadata, host_fqdn):
+    """Extract managed services from a dictionary of metadata
+
+    This filter is useful for parsing server metadata that is loaded on to
+    instances and describes the services that instance will host. The metadata
+    is written to disk as JSON on the instance, but this filter expects a
+    dictionary. You can invoke the filter with the following:
+
+    {{ server_metadata | from_json | parse_service_metadata(host_fqdn) }}
+
+    This filter is useful for dynamically creating service principals in
+    FreeIPA for services running on a specific host, which we can later use to
+    generate TLS certificates. For example:
+
+    - name: parse metadata for services
+      include: register_services.yaml
+      loop: {{ metadata | from_json | parse_service_metadata(host_fqdn) }}
+
+    register_services.yaml
+
+    ---
+    - name: add sub-host in FreeIPA
+      ipa_host:
+        fqdn: {{ item.0 }}
+        state: present
+
+    - name: add service to FreeIPA
+      ipa_service:
+        name: "{{ item.1 }}/{{ sub_host }} "
+        state: present
+
+    :param service_metadata: is a dictionary where keys are strings that
+                             describe the service. The value can be either a
+                             list of networks (compact notation) or a string
+                             that represents the service and principal (managed
+                             notation).
+    :param host_fqdn: is a string that represents the fully-qualified hostname
+                      of the host we're processing metadata for (e.g.,
+                      'controller-0.example.test')
+    :returns: a list of tuples where the first element of the tuple is the
+              fully-qualified domain name of the service (e.g.,
+              'controller-0.external.example.test') and the second element is
+              the service (e.g., 'haproxy').
+    """
+    hostname = host_fqdn.split('.')[0]
+    domain = host_fqdn.split('.', 1)[1]
+    managed_services = set()
+    for service_key in service_metadata.keys():
+        if service_key.startswith('managed_service_'):
+            principal = service_metadata[service_key]
+            service_name, service_hostname = principal.split('/', 2)
+            managed_services.add((service_hostname, service_name))
+        elif service_key.startswith('compact_service_'):
+            interfaces = service_metadata[service_key]
+            service_name = service_key.split('_', 2)[-1]
+            for interface in interfaces:
+                service_hostname = '.'.join([hostname, interface, domain])
+                managed_services.add((service_hostname, service_name))
+
+    return list(managed_services)
+
+
+class FilterModule(object):
+    def filters(self):
+        return {'parse_service_metadata': parse_service_metadata}

tripleo_ipa/tests/base.py

@@ -0,0 +1,35 @@
+# Copyright 2019 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ansible.plugins import loader
+
+from oslotest import base
+
+
+def load_module_utils(*args):
+    """Ensure requested module_utils are loaded into ansible.module_utils"""
+    if args:
+        for m in args:
+            try:
+                loader.module_utils_loader.get(m)
+            except AttributeError:
+                pass
+    else:
+        # search and load all module_utils, its noisy and slower
+        list(loader.module_utils_loader.all())
+
+
+class TestCase(base.BaseTestCase):
+    """Test case base class for all unit tests."""

tripleo_ipa/tests/plugins/filter/test_parse_service_metadata.py

@@ -0,0 +1,135 @@
+# Copyright 2020 Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+
+from tripleo_ipa.ansible_plugins.filter import service_metadata
+from tripleo_ipa.tests import base as tests_base
+
+# Short-hand prefixes
+MS = 'managed_service_'
+CS = 'compact_service_'
+
+
+class TestParseServiceMetadata(tests_base.TestCase):
+
+    def setUp(self):
+        super(TestParseServiceMetadata, self).setUp()
+
+    def test_parse_service_metadata(self):
+
+        domain = 'example.test'
+        host_fqdn = 'test-0.' + domain
+        md = {
+            CS + 'HTTP': [
+                'ctlplane', 'storage', 'storagemgmt', 'internalapi', 'external'
+            ],
+            CS + 'haproxy': ['ctlplane', 'storage', 'storagemgmt', 'internalapi'],
+            CS + 'libvirt-vnc': ['internalapi'],
+            CS + 'mysql': ['internalapi'],
+            CS + 'neutron_ovn': ['internalapi'],
+            CS + 'novnc-proxy': ['internalapi'],
+            CS + 'ovn_controller': ['internalapi'],
+            CS + 'ovn_dbs': ['internalapi'],
+            CS + 'rabbitmq': ['internalapi'],
+            CS + 'redis': ['internalapi'],
+            MS + 'haproxyctlplane': 'haproxy/test-0.ctlplane.' + domain,
+            MS + 'haproxyexternal': 'haproxy/test-0.' + domain,
+            MS + 'haproxyinternal_api': 'haproxy/test-0.internalapi.' + domain,
+            MS + 'haproxystorage': 'haproxy/test-0.storage.' + domain,
+            MS + 'haproxystorage_mgmt': 'haproxy/test-0.storagemgmt.' + domain,
+            MS + 'mysqlinternal_api': 'mysql/test-0.internalapi.' + domain,
+            MS + 'ovn_dbsinternal_api': 'ovn_dbs/test-0.internalapi.' + domain,
+            MS + 'redisinternal_api': 'redis/test-0.internalapi.' + domain
+        }
+
+        expected_services = [
+            ('test-0.ctlplane.example.test', 'HTTP'),
+            ('test-0.storage.example.test', 'HTTP'),
+            ('test-0.storagemgmt.example.test', 'HTTP'),
+            ('test-0.internalapi.example.test', 'HTTP'),
+            ('test-0.external.example.test', 'HTTP'),
+            ('test-0.ctlplane.example.test', 'haproxy'),
+            ('test-0.example.test', 'haproxy'),
+            ('test-0.internalapi.example.test', 'haproxy'),
+            ('test-0.storage.example.test', 'haproxy'),
+            ('test-0.storagemgmt.example.test', 'haproxy'),
+            ('test-0.internalapi.example.test', 'libvirt-vnc'),
+            ('test-0.internalapi.example.test', 'mysql'),
+            ('test-0.internalapi.example.test', 'neutron_ovn'),
+            ('test-0.internalapi.example.test', 'novnc-proxy'),
+            ('test-0.internalapi.example.test', 'ovn_controller'),
+            ('test-0.internalapi.example.test', 'ovn_dbs'),
+            ('test-0.internalapi.example.test', 'rabbitmq'),
+            ('test-0.internalapi.example.test', 'redis')
+        ]
+
+        services = service_metadata.parse_service_metadata(md, host_fqdn)
+        self.assertEqual(len(services), len(expected_services))
+        for service in services:
+            self.assertIn(service, expected_services)
+
+    def test_parse_service_metadata_with_long_domain_name(self):
+
+        domain = 'cloud.example.test'
+        host_fqdn = 'test-0.' + domain
+        md = {
+            CS + 'HTTP': [
+                'ctlplane', 'storage', 'storagemgmt', 'internalapi', 'external'
+            ],
+            CS + 'haproxy': ['ctlplane', 'storage', 'storagemgmt', 'internalapi'],
+            CS + 'libvirt-vnc': ['internalapi'],
+            CS + 'mysql': ['internalapi'],
+            CS + 'neutron_ovn': ['internalapi'],
+            CS + 'novnc-proxy': ['internalapi'],
+            CS + 'ovn_controller': ['internalapi'],
+            CS + 'ovn_dbs': ['internalapi'],
+            CS + 'rabbitmq': ['internalapi'],
+            CS + 'redis': ['internalapi'],
+            MS + 'haproxyctlplane': 'haproxy/test-0.ctlplane.' + domain,
+            MS + 'haproxyexternal': 'haproxy/test-0.' + domain,
+            MS + 'haproxyinternal_api': 'haproxy/test-0.internalapi.' + domain,
+            MS + 'haproxystorage': 'haproxy/test-0.storage.' + domain,
+            MS + 'haproxystorage_mgmt': 'haproxy/test-0.storagemgmt.' + domain,
+            MS + 'mysqlinternal_api': 'mysql/test-0.internalapi.' + domain,
+            MS + 'ovn_dbsinternal_api': 'ovn_dbs/test-0.internalapi.' + domain,
+            MS + 'redisinternal_api': 'redis/test-0.internalapi.' + domain
+        }
+
+        expected_services = [
+            ('test-0.ctlplane.cloud.example.test', 'HTTP'),
+            ('test-0.storage.cloud.example.test', 'HTTP'),
+            ('test-0.storagemgmt.cloud.example.test', 'HTTP'),
+            ('test-0.internalapi.cloud.example.test', 'HTTP'),
+            ('test-0.external.cloud.example.test', 'HTTP'),
+            ('test-0.ctlplane.cloud.example.test', 'haproxy'),
+            ('test-0.cloud.example.test', 'haproxy'),
+            ('test-0.internalapi.cloud.example.test', 'haproxy'),
+            ('test-0.storage.cloud.example.test', 'haproxy'),
+            ('test-0.storagemgmt.cloud.example.test', 'haproxy'),
+            ('test-0.internalapi.cloud.example.test', 'libvirt-vnc'),
+            ('test-0.internalapi.cloud.example.test', 'mysql'),
+            ('test-0.internalapi.cloud.example.test', 'neutron_ovn'),
+            ('test-0.internalapi.cloud.example.test', 'novnc-proxy'),
+            ('test-0.internalapi.cloud.example.test', 'ovn_controller'),
+            ('test-0.internalapi.cloud.example.test', 'ovn_dbs'),
+            ('test-0.internalapi.cloud.example.test', 'rabbitmq'),
+            ('test-0.internalapi.cloud.example.test', 'redis')
+        ]
+
+        services = service_metadata.parse_service_metadata(md, host_fqdn)
+        self.assertEqual(len(services), len(expected_services))
+        for service in services:
+            self.assertIn(service, expected_services)

zuul.d/layout.yaml

@@ -1,5 +1,9 @@
 ---
 - project:
+    templates:
+      - check-requirements
+      - openstack-python36-jobs
+      - publish-to-pypi
     check:
       jobs:
       - tripleo-ipa-centos-8-molecule