From a3d1af2413109c00485a9453254548e1fa8a169d Mon Sep 17 00:00:00 2001 From: Lance Bragstad Date: Fri, 6 Mar 2020 12:56:31 -0600 Subject: [PATCH] Add a custom filter for parsing instance metadata TripleO's TLS implementation using nova in the undercloud relies on instance metadata that contains the services a particular instance is going to host. For example, metadata for a controller (e.g., controller-0) will contain JSON that describes the services and interfaces that are supposed to be hosted on controller (e.g., haproxy/controller-0.ctlplane.example.test). For TLS, we need to generate certificates for those services automatically. This requires us to parse the metadata for the instance, determine the services that are supposed to the hosted by that instance, and then curate unique principals that we add to FreeIPA. Once the appropriate hosts (e.g., controller-0.example.test), sub-hosts (e.g., controller-0.ctlplane.example.test), and principals (e.g., haproxy/controller-0.ctlplane.example.test) have been added to FreeIPA, we can generate certificates for that service and configure it to use TLS. The commit attempts to introduce a filter that parses metadata from the instance and returns a list of tuples to the caller. Each tuple is unique to each service being hosted on an instance and we use them to generate the requisite entities in FreeIPA for TLS certificates using ansible-freeipa later. The functionality was formally a part of novajoin, which is a nova metadata service. This patch adds unit tests and relevant env configuration. Change-Id: Ibbac29a9d2570280cad0f426ad0b0367587bfee7 --- .pre-commit-config.yaml | 49 +++++++ .stestr.conf | 3 + ansible-requirements.txt | 1 + setup.cfg | 20 ++- test-requirements.txt | 11 +- tox.ini | 17 ++- .../filter/service_metadata.py | 82 +++++++++++ tripleo_ipa/tests/__init__.py | 0 tripleo_ipa/tests/base.py | 35 +++++ tripleo_ipa/tests/plugins/__init__.py | 0 tripleo_ipa/tests/plugins/filter/__init__.py | 0 .../filter/test_parse_service_metadata.py | 135 ++++++++++++++++++ zuul.d/layout.yaml | 4 + 13 files changed, 350 insertions(+), 7 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 .stestr.conf create mode 100644 ansible-requirements.txt create mode 100644 tripleo_ipa/ansible_plugins/filter/service_metadata.py create mode 100644 tripleo_ipa/tests/__init__.py create mode 100644 tripleo_ipa/tests/base.py create mode 100644 tripleo_ipa/tests/plugins/__init__.py create mode 100644 tripleo_ipa/tests/plugins/filter/__init__.py create mode 100644 tripleo_ipa/tests/plugins/filter/test_parse_service_metadata.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..0ed5424 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,49 @@ +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.1.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: mixed-line-ending + - id: check-byte-order-marker + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: debug-statements + - id: flake8 + entry: flake8 --ignore=E24,E121,E122,E123,E124,E126,E226,E265,E305,E402,F401,F405,E501,E704,F403,F841,W503 + # TODO(cloudnull): These codes were added to pass the lint check. + # All of these ignore codes should be resolved in + # future PRs. + - id: check-yaml + files: .*\.(yaml|yml)$ + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.15.0 + hooks: + - id: yamllint + files: \.(yaml|yml)$ + types: [file, yaml] + entry: yamllint --strict -f parsable + - repo: https://github.com/ansible/ansible-lint + rev: v4.1.1a2 + hooks: + - id: ansible-lint + files: \.(yaml|yml)$ + entry: >- + ansible-lint --force-color -v -x "ANSIBLE0006,ANSIBLE0007,ANSIBLE0010,ANSIBLE0012,ANSIBLE0013,ANSIBLE0016" + --exclude=tripleo_ansible/roles.galaxy + # TODO(cloudnull): These codes were added to pass the lint check. + # Things found within roles.galaxy are external + # and not something maintained here. + - repo: https://github.com/openstack-dev/bashate.git + rev: 0.6.0 + hooks: + - id: bashate + entry: bashate --error . --verbose --ignore=E006,E040 + # Run bashate check for all bash scripts + # Ignores the following rules: + # E006: Line longer than 79 columns (as many scripts use jinja + # templating, this is very difficult) + # E040: Syntax error determined using `bash -n` (as many scripts + # use jinja templating, this will often fail and the syntax + # error will be discovered in execution anyway) diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 0000000..a10a38c --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=${TEST_PATH:-./tripleo_ipa/tests/} +top_dir=./ diff --git a/ansible-requirements.txt b/ansible-requirements.txt new file mode 100644 index 0000000..a89eb19 --- /dev/null +++ b/ansible-requirements.txt @@ -0,0 +1 @@ +ansible>=2.8 diff --git a/setup.cfg b/setup.cfg index 59e56c2..8a9202a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -name = tripleo_ipa +name = tripleo-ipa summary = Ansible assets for interacting with FreeIPA on behalf of TripleO description-file = README.rst @@ -8,9 +8,25 @@ author = RedHat home-page = https://opendev.org/x/tripleo-ipa classifier = License :: OSI Approved :: Apache Software License - Development Status :: 2 - Pre-Alpha + Development Status :: 4 - Beta Intended Audience :: Developers Intended Audience :: System Administrators Intended Audience :: Information Technology Topic :: Utilities +[global] +setup-hooks = + pbr.hooks.setup_hook + +[files] +data_files = + share/ansible/tripleo-playbooks/ = tripleo_ipa/playbooks/* + share/ansible/plugins/ = tripleo_ipa/ansible_plugins/* + share/ansible/roles/ = tripleo_ipa/roles/* + +[wheel] +universal = 1 + +[pbr] +skip_authors = True +skip_changelog = True diff --git a/test-requirements.txt b/test-requirements.txt index 4c2b5be..64f45bf 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,9 @@ -ansible-lint -yamllint +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +ansible-lint # MIT +pre-commit # MIT +mock>=2.0.0 # BSD +stestr>=2.0.0 # Apache-2.0 +oslotest>=3.2.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index 15116d2..c16cc4e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] minversion = 2.0 # add docs to the list of environments once we actually have docs to generate -envlist = molecule, linters +envlist = py36,pep8,molecule,linters skipdist = True [testenv] @@ -9,12 +9,18 @@ usedevelop = True install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt} {opts} {packages} passenv = * sitepackages = True +deps = + -r {toxinidir}/ansible-requirements.txt + -r {toxinidir}/test-requirements.txt +commands = stestr run {posargs} +whitelist_externals = + tox + +[testenv:molecule] deps = -r {toxinidir}/molecule-requirements.txt changedir = {toxinidir}/tripleo_ipa commands = molecule test --all -whitelist_externals = - tox [testenv:ansible-lint] deps = {[testenv:linters]deps} @@ -32,3 +38,8 @@ deps = commands = {[testenv:ansible-lint]commands} {[testenv:yamllint]commands} + +[testenv:pep8] +envdir = {toxworkdir}/linters +commands = + python -m pre_commit run flake8 -a diff --git a/tripleo_ipa/ansible_plugins/filter/service_metadata.py b/tripleo_ipa/ansible_plugins/filter/service_metadata.py new file mode 100644 index 0000000..66c657c --- /dev/null +++ b/tripleo_ipa/ansible_plugins/filter/service_metadata.py @@ -0,0 +1,82 @@ +#!/usr/bin/python +# +# Copyright 2020 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def parse_service_metadata(service_metadata, host_fqdn): + """Extract managed services from a dictionary of metadata + + This filter is useful for parsing server metadata that is loaded on to + instances and describes the services that instance will host. The metadata + is written to disk as JSON on the instance, but this filter expects a + dictionary. You can invoke the filter with the following: + + {{ server_metadata | from_json | parse_service_metadata(host_fqdn) }} + + This filter is useful for dynamically creating service principals in + FreeIPA for services running on a specific host, which we can later use to + generate TLS certificates. For example: + + - name: parse metadata for services + include: register_services.yaml + loop: {{ metadata | from_json | parse_service_metadata(host_fqdn) }} + + register_services.yaml + + --- + - name: add sub-host in FreeIPA + ipa_host: + fqdn: {{ item.0 }} + state: present + + - name: add service to FreeIPA + ipa_service: + name: "{{ item.1 }}/{{ sub_host }} " + state: present + + :param service_metadata: is a dictionary where keys are strings that + describe the service. The value can be either a + list of networks (compact notation) or a string + that represents the service and principal (managed + notation). + :param host_fqdn: is a string that represents the fully-qualified hostname + of the host we're processing metadata for (e.g., + 'controller-0.example.test') + :returns: a list of tuples where the first element of the tuple is the + fully-qualified domain name of the service (e.g., + 'controller-0.external.example.test') and the second element is + the service (e.g., 'haproxy'). + """ + hostname = host_fqdn.split('.')[0] + domain = host_fqdn.split('.', 1)[1] + managed_services = set() + for service_key in service_metadata.keys(): + if service_key.startswith('managed_service_'): + principal = service_metadata[service_key] + service_name, service_hostname = principal.split('/', 2) + managed_services.add((service_hostname, service_name)) + elif service_key.startswith('compact_service_'): + interfaces = service_metadata[service_key] + service_name = service_key.split('_', 2)[-1] + for interface in interfaces: + service_hostname = '.'.join([hostname, interface, domain]) + managed_services.add((service_hostname, service_name)) + + return list(managed_services) + + +class FilterModule(object): + def filters(self): + return {'parse_service_metadata': parse_service_metadata} diff --git a/tripleo_ipa/tests/__init__.py b/tripleo_ipa/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tripleo_ipa/tests/base.py b/tripleo_ipa/tests/base.py new file mode 100644 index 0000000..b55e760 --- /dev/null +++ b/tripleo_ipa/tests/base.py @@ -0,0 +1,35 @@ +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from ansible.plugins import loader + +from oslotest import base + + +def load_module_utils(*args): + """Ensure requested module_utils are loaded into ansible.module_utils""" + if args: + for m in args: + try: + loader.module_utils_loader.get(m) + except AttributeError: + pass + else: + # search and load all module_utils, its noisy and slower + list(loader.module_utils_loader.all()) + + +class TestCase(base.BaseTestCase): + """Test case base class for all unit tests.""" diff --git a/tripleo_ipa/tests/plugins/__init__.py b/tripleo_ipa/tests/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tripleo_ipa/tests/plugins/filter/__init__.py b/tripleo_ipa/tests/plugins/filter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tripleo_ipa/tests/plugins/filter/test_parse_service_metadata.py b/tripleo_ipa/tests/plugins/filter/test_parse_service_metadata.py new file mode 100644 index 0000000..d89721e --- /dev/null +++ b/tripleo_ipa/tests/plugins/filter/test_parse_service_metadata.py @@ -0,0 +1,135 @@ +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +from tripleo_ipa.ansible_plugins.filter import service_metadata +from tripleo_ipa.tests import base as tests_base + +# Short-hand prefixes +MS = 'managed_service_' +CS = 'compact_service_' + + +class TestParseServiceMetadata(tests_base.TestCase): + + def setUp(self): + super(TestParseServiceMetadata, self).setUp() + + def test_parse_service_metadata(self): + + domain = 'example.test' + host_fqdn = 'test-0.' + domain + md = { + CS + 'HTTP': [ + 'ctlplane', 'storage', 'storagemgmt', 'internalapi', 'external' + ], + CS + 'haproxy': ['ctlplane', 'storage', 'storagemgmt', 'internalapi'], + CS + 'libvirt-vnc': ['internalapi'], + CS + 'mysql': ['internalapi'], + CS + 'neutron_ovn': ['internalapi'], + CS + 'novnc-proxy': ['internalapi'], + CS + 'ovn_controller': ['internalapi'], + CS + 'ovn_dbs': ['internalapi'], + CS + 'rabbitmq': ['internalapi'], + CS + 'redis': ['internalapi'], + MS + 'haproxyctlplane': 'haproxy/test-0.ctlplane.' + domain, + MS + 'haproxyexternal': 'haproxy/test-0.' + domain, + MS + 'haproxyinternal_api': 'haproxy/test-0.internalapi.' + domain, + MS + 'haproxystorage': 'haproxy/test-0.storage.' + domain, + MS + 'haproxystorage_mgmt': 'haproxy/test-0.storagemgmt.' + domain, + MS + 'mysqlinternal_api': 'mysql/test-0.internalapi.' + domain, + MS + 'ovn_dbsinternal_api': 'ovn_dbs/test-0.internalapi.' + domain, + MS + 'redisinternal_api': 'redis/test-0.internalapi.' + domain + } + + expected_services = [ + ('test-0.ctlplane.example.test', 'HTTP'), + ('test-0.storage.example.test', 'HTTP'), + ('test-0.storagemgmt.example.test', 'HTTP'), + ('test-0.internalapi.example.test', 'HTTP'), + ('test-0.external.example.test', 'HTTP'), + ('test-0.ctlplane.example.test', 'haproxy'), + ('test-0.example.test', 'haproxy'), + ('test-0.internalapi.example.test', 'haproxy'), + ('test-0.storage.example.test', 'haproxy'), + ('test-0.storagemgmt.example.test', 'haproxy'), + ('test-0.internalapi.example.test', 'libvirt-vnc'), + ('test-0.internalapi.example.test', 'mysql'), + ('test-0.internalapi.example.test', 'neutron_ovn'), + ('test-0.internalapi.example.test', 'novnc-proxy'), + ('test-0.internalapi.example.test', 'ovn_controller'), + ('test-0.internalapi.example.test', 'ovn_dbs'), + ('test-0.internalapi.example.test', 'rabbitmq'), + ('test-0.internalapi.example.test', 'redis') + ] + + services = service_metadata.parse_service_metadata(md, host_fqdn) + self.assertEqual(len(services), len(expected_services)) + for service in services: + self.assertIn(service, expected_services) + + def test_parse_service_metadata_with_long_domain_name(self): + + domain = 'cloud.example.test' + host_fqdn = 'test-0.' + domain + md = { + CS + 'HTTP': [ + 'ctlplane', 'storage', 'storagemgmt', 'internalapi', 'external' + ], + CS + 'haproxy': ['ctlplane', 'storage', 'storagemgmt', 'internalapi'], + CS + 'libvirt-vnc': ['internalapi'], + CS + 'mysql': ['internalapi'], + CS + 'neutron_ovn': ['internalapi'], + CS + 'novnc-proxy': ['internalapi'], + CS + 'ovn_controller': ['internalapi'], + CS + 'ovn_dbs': ['internalapi'], + CS + 'rabbitmq': ['internalapi'], + CS + 'redis': ['internalapi'], + MS + 'haproxyctlplane': 'haproxy/test-0.ctlplane.' + domain, + MS + 'haproxyexternal': 'haproxy/test-0.' + domain, + MS + 'haproxyinternal_api': 'haproxy/test-0.internalapi.' + domain, + MS + 'haproxystorage': 'haproxy/test-0.storage.' + domain, + MS + 'haproxystorage_mgmt': 'haproxy/test-0.storagemgmt.' + domain, + MS + 'mysqlinternal_api': 'mysql/test-0.internalapi.' + domain, + MS + 'ovn_dbsinternal_api': 'ovn_dbs/test-0.internalapi.' + domain, + MS + 'redisinternal_api': 'redis/test-0.internalapi.' + domain + } + + expected_services = [ + ('test-0.ctlplane.cloud.example.test', 'HTTP'), + ('test-0.storage.cloud.example.test', 'HTTP'), + ('test-0.storagemgmt.cloud.example.test', 'HTTP'), + ('test-0.internalapi.cloud.example.test', 'HTTP'), + ('test-0.external.cloud.example.test', 'HTTP'), + ('test-0.ctlplane.cloud.example.test', 'haproxy'), + ('test-0.cloud.example.test', 'haproxy'), + ('test-0.internalapi.cloud.example.test', 'haproxy'), + ('test-0.storage.cloud.example.test', 'haproxy'), + ('test-0.storagemgmt.cloud.example.test', 'haproxy'), + ('test-0.internalapi.cloud.example.test', 'libvirt-vnc'), + ('test-0.internalapi.cloud.example.test', 'mysql'), + ('test-0.internalapi.cloud.example.test', 'neutron_ovn'), + ('test-0.internalapi.cloud.example.test', 'novnc-proxy'), + ('test-0.internalapi.cloud.example.test', 'ovn_controller'), + ('test-0.internalapi.cloud.example.test', 'ovn_dbs'), + ('test-0.internalapi.cloud.example.test', 'rabbitmq'), + ('test-0.internalapi.cloud.example.test', 'redis') + ] + + services = service_metadata.parse_service_metadata(md, host_fqdn) + self.assertEqual(len(services), len(expected_services)) + for service in services: + self.assertIn(service, expected_services) diff --git a/zuul.d/layout.yaml b/zuul.d/layout.yaml index fe42d63..848b30b 100644 --- a/zuul.d/layout.yaml +++ b/zuul.d/layout.yaml @@ -1,5 +1,9 @@ --- - project: + templates: + - check-requirements + - openstack-python36-jobs + - publish-to-pypi check: jobs: - tripleo-ipa-centos-8-molecule