diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..0ed5424 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,49 @@ +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.1.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: mixed-line-ending + - id: check-byte-order-marker + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: debug-statements + - id: flake8 + entry: flake8 --ignore=E24,E121,E122,E123,E124,E126,E226,E265,E305,E402,F401,F405,E501,E704,F403,F841,W503 + # TODO(cloudnull): These codes were added to pass the lint check. + # All of these ignore codes should be resolved in + # future PRs. + - id: check-yaml + files: .*\.(yaml|yml)$ + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.15.0 + hooks: + - id: yamllint + files: \.(yaml|yml)$ + types: [file, yaml] + entry: yamllint --strict -f parsable + - repo: https://github.com/ansible/ansible-lint + rev: v4.1.1a2 + hooks: + - id: ansible-lint + files: \.(yaml|yml)$ + entry: >- + ansible-lint --force-color -v -x "ANSIBLE0006,ANSIBLE0007,ANSIBLE0010,ANSIBLE0012,ANSIBLE0013,ANSIBLE0016" + --exclude=tripleo_ansible/roles.galaxy + # TODO(cloudnull): These codes were added to pass the lint check. + # Things found within roles.galaxy are external + # and not something maintained here. + - repo: https://github.com/openstack-dev/bashate.git + rev: 0.6.0 + hooks: + - id: bashate + entry: bashate --error . --verbose --ignore=E006,E040 + # Run bashate check for all bash scripts + # Ignores the following rules: + # E006: Line longer than 79 columns (as many scripts use jinja + # templating, this is very difficult) + # E040: Syntax error determined using `bash -n` (as many scripts + # use jinja templating, this will often fail and the syntax + # error will be discovered in execution anyway) diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 0000000..a10a38c --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=${TEST_PATH:-./tripleo_ipa/tests/} +top_dir=./ diff --git a/ansible-requirements.txt b/ansible-requirements.txt new file mode 100644 index 0000000..a89eb19 --- /dev/null +++ b/ansible-requirements.txt @@ -0,0 +1 @@ +ansible>=2.8 diff --git a/setup.cfg b/setup.cfg index 59e56c2..8a9202a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -name = tripleo_ipa +name = tripleo-ipa summary = Ansible assets for interacting with FreeIPA on behalf of TripleO description-file = README.rst @@ -8,9 +8,25 @@ author = RedHat home-page = https://opendev.org/x/tripleo-ipa classifier = License :: OSI Approved :: Apache Software License - Development Status :: 2 - Pre-Alpha + Development Status :: 4 - Beta Intended Audience :: Developers Intended Audience :: System Administrators Intended Audience :: Information Technology Topic :: Utilities +[global] +setup-hooks = + pbr.hooks.setup_hook + +[files] +data_files = + share/ansible/tripleo-playbooks/ = tripleo_ipa/playbooks/* + share/ansible/plugins/ = tripleo_ipa/ansible_plugins/* + share/ansible/roles/ = tripleo_ipa/roles/* + +[wheel] +universal = 1 + +[pbr] +skip_authors = True +skip_changelog = True diff --git a/test-requirements.txt b/test-requirements.txt index 4c2b5be..64f45bf 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,9 @@ -ansible-lint -yamllint +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +ansible-lint # MIT +pre-commit # MIT +mock>=2.0.0 # BSD +stestr>=2.0.0 # Apache-2.0 +oslotest>=3.2.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index 15116d2..c16cc4e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] minversion = 2.0 # add docs to the list of environments once we actually have docs to generate -envlist = molecule, linters +envlist = py36,pep8,molecule,linters skipdist = True [testenv] @@ -9,12 +9,18 @@ usedevelop = True install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt} {opts} {packages} passenv = * sitepackages = True +deps = + -r {toxinidir}/ansible-requirements.txt + -r {toxinidir}/test-requirements.txt +commands = stestr run {posargs} +whitelist_externals = + tox + +[testenv:molecule] deps = -r {toxinidir}/molecule-requirements.txt changedir = {toxinidir}/tripleo_ipa commands = molecule test --all -whitelist_externals = - tox [testenv:ansible-lint] deps = {[testenv:linters]deps} @@ -32,3 +38,8 @@ deps = commands = {[testenv:ansible-lint]commands} {[testenv:yamllint]commands} + +[testenv:pep8] +envdir = {toxworkdir}/linters +commands = + python -m pre_commit run flake8 -a diff --git a/tripleo_ipa/ansible_plugins/filter/service_metadata.py b/tripleo_ipa/ansible_plugins/filter/service_metadata.py new file mode 100644 index 0000000..66c657c --- /dev/null +++ b/tripleo_ipa/ansible_plugins/filter/service_metadata.py @@ -0,0 +1,82 @@ +#!/usr/bin/python +# +# Copyright 2020 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def parse_service_metadata(service_metadata, host_fqdn): + """Extract managed services from a dictionary of metadata + + This filter is useful for parsing server metadata that is loaded on to + instances and describes the services that instance will host. The metadata + is written to disk as JSON on the instance, but this filter expects a + dictionary. You can invoke the filter with the following: + + {{ server_metadata | from_json | parse_service_metadata(host_fqdn) }} + + This filter is useful for dynamically creating service principals in + FreeIPA for services running on a specific host, which we can later use to + generate TLS certificates. For example: + + - name: parse metadata for services + include: register_services.yaml + loop: {{ metadata | from_json | parse_service_metadata(host_fqdn) }} + + register_services.yaml + + --- + - name: add sub-host in FreeIPA + ipa_host: + fqdn: {{ item.0 }} + state: present + + - name: add service to FreeIPA + ipa_service: + name: "{{ item.1 }}/{{ sub_host }} " + state: present + + :param service_metadata: is a dictionary where keys are strings that + describe the service. The value can be either a + list of networks (compact notation) or a string + that represents the service and principal (managed + notation). + :param host_fqdn: is a string that represents the fully-qualified hostname + of the host we're processing metadata for (e.g., + 'controller-0.example.test') + :returns: a list of tuples where the first element of the tuple is the + fully-qualified domain name of the service (e.g., + 'controller-0.external.example.test') and the second element is + the service (e.g., 'haproxy'). + """ + hostname = host_fqdn.split('.')[0] + domain = host_fqdn.split('.', 1)[1] + managed_services = set() + for service_key in service_metadata.keys(): + if service_key.startswith('managed_service_'): + principal = service_metadata[service_key] + service_name, service_hostname = principal.split('/', 2) + managed_services.add((service_hostname, service_name)) + elif service_key.startswith('compact_service_'): + interfaces = service_metadata[service_key] + service_name = service_key.split('_', 2)[-1] + for interface in interfaces: + service_hostname = '.'.join([hostname, interface, domain]) + managed_services.add((service_hostname, service_name)) + + return list(managed_services) + + +class FilterModule(object): + def filters(self): + return {'parse_service_metadata': parse_service_metadata} diff --git a/tripleo_ipa/tests/__init__.py b/tripleo_ipa/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tripleo_ipa/tests/base.py b/tripleo_ipa/tests/base.py new file mode 100644 index 0000000..b55e760 --- /dev/null +++ b/tripleo_ipa/tests/base.py @@ -0,0 +1,35 @@ +# Copyright 2019 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from ansible.plugins import loader + +from oslotest import base + + +def load_module_utils(*args): + """Ensure requested module_utils are loaded into ansible.module_utils""" + if args: + for m in args: + try: + loader.module_utils_loader.get(m) + except AttributeError: + pass + else: + # search and load all module_utils, its noisy and slower + list(loader.module_utils_loader.all()) + + +class TestCase(base.BaseTestCase): + """Test case base class for all unit tests.""" diff --git a/tripleo_ipa/tests/plugins/__init__.py b/tripleo_ipa/tests/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tripleo_ipa/tests/plugins/filter/__init__.py b/tripleo_ipa/tests/plugins/filter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tripleo_ipa/tests/plugins/filter/test_parse_service_metadata.py b/tripleo_ipa/tests/plugins/filter/test_parse_service_metadata.py new file mode 100644 index 0000000..d89721e --- /dev/null +++ b/tripleo_ipa/tests/plugins/filter/test_parse_service_metadata.py @@ -0,0 +1,135 @@ +# Copyright 2020 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +from tripleo_ipa.ansible_plugins.filter import service_metadata +from tripleo_ipa.tests import base as tests_base + +# Short-hand prefixes +MS = 'managed_service_' +CS = 'compact_service_' + + +class TestParseServiceMetadata(tests_base.TestCase): + + def setUp(self): + super(TestParseServiceMetadata, self).setUp() + + def test_parse_service_metadata(self): + + domain = 'example.test' + host_fqdn = 'test-0.' + domain + md = { + CS + 'HTTP': [ + 'ctlplane', 'storage', 'storagemgmt', 'internalapi', 'external' + ], + CS + 'haproxy': ['ctlplane', 'storage', 'storagemgmt', 'internalapi'], + CS + 'libvirt-vnc': ['internalapi'], + CS + 'mysql': ['internalapi'], + CS + 'neutron_ovn': ['internalapi'], + CS + 'novnc-proxy': ['internalapi'], + CS + 'ovn_controller': ['internalapi'], + CS + 'ovn_dbs': ['internalapi'], + CS + 'rabbitmq': ['internalapi'], + CS + 'redis': ['internalapi'], + MS + 'haproxyctlplane': 'haproxy/test-0.ctlplane.' + domain, + MS + 'haproxyexternal': 'haproxy/test-0.' + domain, + MS + 'haproxyinternal_api': 'haproxy/test-0.internalapi.' + domain, + MS + 'haproxystorage': 'haproxy/test-0.storage.' + domain, + MS + 'haproxystorage_mgmt': 'haproxy/test-0.storagemgmt.' + domain, + MS + 'mysqlinternal_api': 'mysql/test-0.internalapi.' + domain, + MS + 'ovn_dbsinternal_api': 'ovn_dbs/test-0.internalapi.' + domain, + MS + 'redisinternal_api': 'redis/test-0.internalapi.' + domain + } + + expected_services = [ + ('test-0.ctlplane.example.test', 'HTTP'), + ('test-0.storage.example.test', 'HTTP'), + ('test-0.storagemgmt.example.test', 'HTTP'), + ('test-0.internalapi.example.test', 'HTTP'), + ('test-0.external.example.test', 'HTTP'), + ('test-0.ctlplane.example.test', 'haproxy'), + ('test-0.example.test', 'haproxy'), + ('test-0.internalapi.example.test', 'haproxy'), + ('test-0.storage.example.test', 'haproxy'), + ('test-0.storagemgmt.example.test', 'haproxy'), + ('test-0.internalapi.example.test', 'libvirt-vnc'), + ('test-0.internalapi.example.test', 'mysql'), + ('test-0.internalapi.example.test', 'neutron_ovn'), + ('test-0.internalapi.example.test', 'novnc-proxy'), + ('test-0.internalapi.example.test', 'ovn_controller'), + ('test-0.internalapi.example.test', 'ovn_dbs'), + ('test-0.internalapi.example.test', 'rabbitmq'), + ('test-0.internalapi.example.test', 'redis') + ] + + services = service_metadata.parse_service_metadata(md, host_fqdn) + self.assertEqual(len(services), len(expected_services)) + for service in services: + self.assertIn(service, expected_services) + + def test_parse_service_metadata_with_long_domain_name(self): + + domain = 'cloud.example.test' + host_fqdn = 'test-0.' + domain + md = { + CS + 'HTTP': [ + 'ctlplane', 'storage', 'storagemgmt', 'internalapi', 'external' + ], + CS + 'haproxy': ['ctlplane', 'storage', 'storagemgmt', 'internalapi'], + CS + 'libvirt-vnc': ['internalapi'], + CS + 'mysql': ['internalapi'], + CS + 'neutron_ovn': ['internalapi'], + CS + 'novnc-proxy': ['internalapi'], + CS + 'ovn_controller': ['internalapi'], + CS + 'ovn_dbs': ['internalapi'], + CS + 'rabbitmq': ['internalapi'], + CS + 'redis': ['internalapi'], + MS + 'haproxyctlplane': 'haproxy/test-0.ctlplane.' + domain, + MS + 'haproxyexternal': 'haproxy/test-0.' + domain, + MS + 'haproxyinternal_api': 'haproxy/test-0.internalapi.' + domain, + MS + 'haproxystorage': 'haproxy/test-0.storage.' + domain, + MS + 'haproxystorage_mgmt': 'haproxy/test-0.storagemgmt.' + domain, + MS + 'mysqlinternal_api': 'mysql/test-0.internalapi.' + domain, + MS + 'ovn_dbsinternal_api': 'ovn_dbs/test-0.internalapi.' + domain, + MS + 'redisinternal_api': 'redis/test-0.internalapi.' + domain + } + + expected_services = [ + ('test-0.ctlplane.cloud.example.test', 'HTTP'), + ('test-0.storage.cloud.example.test', 'HTTP'), + ('test-0.storagemgmt.cloud.example.test', 'HTTP'), + ('test-0.internalapi.cloud.example.test', 'HTTP'), + ('test-0.external.cloud.example.test', 'HTTP'), + ('test-0.ctlplane.cloud.example.test', 'haproxy'), + ('test-0.cloud.example.test', 'haproxy'), + ('test-0.internalapi.cloud.example.test', 'haproxy'), + ('test-0.storage.cloud.example.test', 'haproxy'), + ('test-0.storagemgmt.cloud.example.test', 'haproxy'), + ('test-0.internalapi.cloud.example.test', 'libvirt-vnc'), + ('test-0.internalapi.cloud.example.test', 'mysql'), + ('test-0.internalapi.cloud.example.test', 'neutron_ovn'), + ('test-0.internalapi.cloud.example.test', 'novnc-proxy'), + ('test-0.internalapi.cloud.example.test', 'ovn_controller'), + ('test-0.internalapi.cloud.example.test', 'ovn_dbs'), + ('test-0.internalapi.cloud.example.test', 'rabbitmq'), + ('test-0.internalapi.cloud.example.test', 'redis') + ] + + services = service_metadata.parse_service_metadata(md, host_fqdn) + self.assertEqual(len(services), len(expected_services)) + for service in services: + self.assertIn(service, expected_services) diff --git a/zuul.d/layout.yaml b/zuul.d/layout.yaml index fe42d63..848b30b 100644 --- a/zuul.d/layout.yaml +++ b/zuul.d/layout.yaml @@ -1,5 +1,9 @@ --- - project: + templates: + - check-requirements + - openstack-python36-jobs + - publish-to-pypi check: jobs: - tripleo-ipa-centos-8-molecule