diff --git a/ansible/kolla-ansible.yml b/ansible/kolla-ansible.yml index 6c1361483..bc8026a6f 100644 --- a/ansible/kolla-ansible.yml +++ b/ansible/kolla-ansible.yml @@ -51,19 +51,6 @@ - kolla_environment_file.stat.exists - kolla_environment != kayobe_environment - # Configuration of extra user-provided Kolla globals. - - name: Check whether a Kolla extra globals configuration file exists - stat: - path: "{{ kayobe_env_config_path ~ '/kolla/globals.yml' }}" - get_checksum: False - get_md5: False - mime: False - register: globals_stat - - - name: Read the Kolla extra globals configuration file - set_fact: - kolla_extra_globals: "{{ lookup('template', kayobe_env_config_path ~ '/kolla/globals.yml') | from_yaml }}" - when: globals_stat.stat.exists tags: - config @@ -117,6 +104,9 @@ kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}" kolla_enable_host_ntp: false docker_daemon_mtu: "{{ public_net_name | net_mtu | default }}" + kolla_globals_paths_extra: + - "{{ kayobe_config_path }}" + - "{{ kayobe_env_config_path }}" - name: Generate Kolla Ansible host vars for the seed host hosts: seed diff --git a/ansible/kolla-bifrost.yml b/ansible/kolla-bifrost.yml index 63e8dca0c..2ad73b958 100644 --- a/ansible/kolla-bifrost.yml +++ b/ansible/kolla-bifrost.yml @@ -3,23 +3,6 @@ hosts: localhost tags: - kolla-bifrost - vars: - kolla_bifrost_extra_globals_path: "{{ kayobe_env_config_path ~ '/kolla/config/bifrost/bifrost.yml' }}" - - pre_tasks: - - name: Check whether a Kolla Bifrost extra globals configuration file exists - stat: - path: "{{ kolla_bifrost_extra_globals_path }}" - get_checksum: False - get_md5: False - mime: False - register: globals_stat - - - name: Read the Kolla Bifrost extra globals configuration file - set_fact: - kolla_bifrost_extra_globals: "{{ lookup('template', kolla_bifrost_extra_globals_path) | from_yaml }}" - when: globals_stat.stat.exists - roles: - role: kolla-bifrost @@ -30,3 +13,6 @@ kolla_bifrost_dnsmasq_dns_servers: "{{ resolv_nameservers | default([]) }}" kolla_bifrost_domain: "{{ resolv_domain | default }}" kolla_bifrost_download_ipa: "{{ not ipa_build_images | bool }}" + kolla_bifrost_config_paths_extra: + - "{{ kayobe_config_path }}" + - "{{ kayobe_env_config_path }}" diff --git a/ansible/kolla-build.yml b/ansible/kolla-build.yml index 37fd6dbd8..a02a533b1 100644 --- a/ansible/kolla-build.yml +++ b/ansible/kolla-build.yml @@ -7,4 +7,6 @@ - role: kolla kolla_install_epel: "{{ dnf_install_epel }}" - role: kolla-build - kolla_build_extra_config_path: "{{ kayobe_env_config_path }}/kolla/kolla-build.conf" + kolla_build_config_paths_extra: + - "{{ kayobe_config_path }}" + - "{{ kayobe_env_config_path }}" diff --git a/ansible/roles/kolla-ansible/defaults/main.yml b/ansible/roles/kolla-ansible/defaults/main.yml index 33b0d2def..02bf8c729 100644 --- a/ansible/roles/kolla-ansible/defaults/main.yml +++ b/ansible/roles/kolla-ansible/defaults/main.yml @@ -231,9 +231,25 @@ kolla_nova_compute_ironic_host: ############################################################################### # Extra free-form configuraton. +# Deprecated: # Free form extra configuration to append to {{ kolla_config_path }}/globals.yml. kolla_extra_globals: +# List of paths to YAML files containing extra configuration to merge to {{ +# kolla_config_path }}/globals.yml. Default is include the globals.yml template +# from the role. +kolla_globals_paths_default: + - "{{ role_path }}/templates" + +# List of paths to YAML files containing extra configuration to merge to {{ +# kolla_config_path }}/globals.yml. Default is an empty list. +kolla_globals_paths_extra: [] + +# List of paths to YAML files containing extra configuration to merge to {{ +# kolla_config_path }}/globals.yml. Default is combination of +# kolla_globals_paths_default and kolla_globals_paths_extra. +kolla_globals_paths: "{{ kolla_globals_paths_default + kolla_globals_paths_extra }}" + # Dictionary containing custom passwords to add or override in the Kolla # passwords file. kolla_ansible_custom_passwords: {} diff --git a/ansible/roles/kolla-ansible/tasks/config.yml b/ansible/roles/kolla-ansible/tasks/config.yml index f74c148a1..276700f01 100644 --- a/ansible/roles/kolla-ansible/tasks/config.yml +++ b/ansible/roles/kolla-ansible/tasks/config.yml @@ -48,8 +48,8 @@ when: (kayobe_environment | default('')) | length > 0 - name: Ensure the Kolla global configuration file exists - template: - src: "globals.yml.j2" + merge_yaml: + sources: "{{ kolla_globals_paths | product(['/kolla/globals.yml']) | map('join') | unique | list }}" dest: "{{ kolla_config_path }}/globals.yml" mode: 0640 vars: diff --git a/ansible/roles/kolla-ansible/templates/globals.yml.j2 b/ansible/roles/kolla-ansible/templates/kolla/globals.yml similarity index 99% rename from ansible/roles/kolla-ansible/templates/globals.yml.j2 rename to ansible/roles/kolla-ansible/templates/kolla/globals.yml index 84de6c1e0..c0f46c237 100644 --- a/ansible/roles/kolla-ansible/templates/globals.yml.j2 +++ b/ansible/roles/kolla-ansible/templates/kolla/globals.yml @@ -1,6 +1,4 @@ --- -# {{ ansible_managed }} - # You can use this file to override _any_ variable throughout Kolla. # Additional options can be found in the # 'kolla-ansible/ansible/group_vars/all.yml' file. Default value of all the diff --git a/ansible/roles/kolla-ansible/tests/main.yml b/ansible/roles/kolla-ansible/tests/main.yml index 8347fa49c..d4b781c96 100644 --- a/ansible/roles/kolla-ansible/tests/main.yml +++ b/ansible/roles/kolla-ansible/tests/main.yml @@ -9,6 +9,7 @@ - import_playbook: test-defaults.yml - import_playbook: test-extras.yml - import_playbook: test-requirements.yml +- import_playbook: test-globals-merge.yml - hosts: localhost connection: local diff --git a/ansible/roles/kolla-ansible/tests/test-defaults.yml b/ansible/roles/kolla-ansible/tests/test-defaults.yml index 3cf9f6efe..f922d8c5a 100644 --- a/ansible/roles/kolla-ansible/tests/test-defaults.yml +++ b/ansible/roles/kolla-ansible/tests/test-defaults.yml @@ -38,6 +38,8 @@ kolla_enable_tls_internal: False kolla_enable_grafana: False kolla_openstack_logging_debug: False + kolla_globals_paths_extra: + - "{{ tempfile_result.path ~ '/etc/kayobe/' }}" apt_cache_valid_time: 3600 - name: Verify kolla-ansible installation diff --git a/ansible/roles/kolla-ansible/tests/test-extras.yml b/ansible/roles/kolla-ansible/tests/test-extras.yml index cde4b9f85..ef3328c87 100644 --- a/ansible/roles/kolla-ansible/tests/test-extras.yml +++ b/ansible/roles/kolla-ansible/tests/test-extras.yml @@ -28,6 +28,14 @@ path: "{{ tempfile_result.path ~ '/etc/kayobe/kolla/inventory/group_vars/foo_group' }}" state: directory + - name: Create extra globals file + copy: + content: | + --- + extra-global-1: "extra-val-1" + extra-global-2: "extra-val-2" + dest: "{{ tempfile_result.path ~ '/etc/kayobe/kolla/globals.yml' }}" + - name: Create custom overcloud foo group vars copy: dest: "{{ tempfile_result.path ~ '/etc/kayobe/kolla/inventory/group_vars/foo_group/all' }}" @@ -181,9 +189,8 @@ kolla_enable_watcher: True kolla_enable_zookeeper: True kolla_enable_zun: True - kolla_extra_globals: - extra-global-1: "extra-val-1" - extra-global-2: "extra-val-2" + kolla_globals_paths_extra: + - "{{ tempfile_result.path ~ '/etc/kayobe/' }}" kolla_ansible_custom_passwords: custom-password-1: "custom-password-1" custom-password-2: "custom-password-2" diff --git a/ansible/roles/kolla-ansible/tests/test-globals-merge.yml b/ansible/roles/kolla-ansible/tests/test-globals-merge.yml new file mode 100644 index 000000000..5110836df --- /dev/null +++ b/ansible/roles/kolla-ansible/tests/test-globals-merge.yml @@ -0,0 +1,116 @@ +--- +- name: Test kolla-ansible role defaults + hosts: localhost + connection: local + tasks: + - name: Create a temporary directory + tempfile: + state: directory + register: tempfile_result + + - block: + + - name: Ensure directories exists + file: + state: directory + mode: "0700" + recurse: true + path: "{{ item }}" + with_items: + - "{{ tempfile_result.path }}/etc/kayobe/kolla/" + - "{{ tempfile_result.path }}//etc/kayobe/environments/level1/kolla" + - "{{ tempfile_result.path }}//etc/kayobe/environments/level2/kolla" + + - name: Write contents to base globals.yml + copy: + content: | + _overridden_level_1_var: base + _base_var: base + dest: "{{ tempfile_result.path }}/etc/kayobe/kolla/globals.yml" + + - name: Write contents to level1 globals.yml + copy: + content: | + _overridden_level_1_var: level1 + _overridden_level_2_var: level1 + dest: "{{ tempfile_result.path }}//etc/kayobe/environments/level1/kolla/globals.yml" + + - name: Write contents to level2 globals.yml + copy: + content: | + _overridden_level_2_var: level2 + dest: "{{ tempfile_result.path }}//etc/kayobe/environments/level2/kolla/globals.yml" + + - name: Test the kolla-ansible role with default values + include_role: + name: ../../kolla-ansible + vars: + kolla_ansible_source_path: "{{ temp_path }}/src" + kolla_ansible_ctl_install_type: "source" + kolla_ansible_source_url: "http://github.com/openstack/kolla-ansible" + kolla_ansible_source_version: "{{ openstack_branch }}" + kolla_ansible_venv: "{{ temp_path }}/venv" + kolla_config_path: "{{ temp_path }}/etc/kolla" + kolla_node_custom_config_path: "{{ temp_path }}/etc/kolla/config" + # Purposely does not exist to simulate the case when no group vars + # are provided + kolla_overcloud_group_vars_path: "{{ temp_path }}/etc/kayobe/kolla/inventory/group_vars" + kolla_ansible_passwords_path: "{{ temp_path }}/passwords.yml" + # Required config. + kolla_base_distro: "fake-distro" + kolla_install_type: "fake-install-type" + kolla_docker_namespace: "fake-namespace" + kolla_openstack_release: "fake-release" + kolla_internal_vip_address: "10.0.0.1" + kolla_internal_fqdn: "fake.internal.fqdn" + kolla_external_vip_address: "10.0.0.2" + kolla_external_fqdn: "fake.external.fqdn" + kolla_ansible_certificates_path: "{{ temp_path }}/etc/kayobe/kolla/certificates" + kolla_enable_tls_external: False + kolla_enable_tls_internal: False + kolla_enable_grafana: False + kolla_openstack_logging_debug: False + kolla_globals_paths_extra: + - "{{ tempfile_result.path ~ '/etc/kayobe/' }}" + - "{{ tempfile_result.path ~ '/etc/kayobe/environments/level1/' }}" + - "{{ tempfile_result.path ~ '/etc/kayobe/environments/level2/' }}" + apt_cache_valid_time: 3600 + + - name: Verify kolla-ansible installation + shell: ". {{ temp_path }}/venv/bin/activate && kolla-ansible -h" + changed_when: False + + - name: Verify ansible installation + command: "{{ temp_path }}/venv/bin/ansible -h" + changed_when: False + + - name: Validate globals.yml contents + assert: + that: + - item.key in globals_yml + - globals_yml[item.key] == item.value + msg: > + Unexpected value for variable "{{ item.key }}" in globals.yml. + Expected "{{ item.value }}", actual + "{{ globals_yml.get(item.key, '') }}". + with_dict: "{{ expected_variables }}" + vars: + # NOTE: Can't use set_fact for this, as it causes kolla-ansible + # Jinja expressions to be evaluated. + globals_yml: "{{ lookup('file', temp_path ~ '/etc/kolla/globals.yml') | from_yaml }}" + expected_variables: + _overridden_level_1_var: level1 + _overridden_level_2_var: level2 + _base_var: base + + always: + - name: Ensure the temporary directory is removed + file: + path: "{{ temp_path }}" + state: absent + rescue: + - name: Flag that a failure occurred + set_fact: + test_failures: "{{ test_failures | default(0) | int + 1 }}" + vars: + temp_path: "{{ tempfile_result.path }}" diff --git a/ansible/roles/kolla-ansible/tests/test-requirements.yml b/ansible/roles/kolla-ansible/tests/test-requirements.yml index c9f8fdb43..67d928f18 100644 --- a/ansible/roles/kolla-ansible/tests/test-requirements.yml +++ b/ansible/roles/kolla-ansible/tests/test-requirements.yml @@ -37,6 +37,8 @@ kolla_enable_tls_internal: False kolla_enable_grafana: False kolla_openstack_logging_debug: False + kolla_globals_paths_extra: + - "{{ tempfile_result.path ~ '/etc/kayobe/' }}" apt_cache_valid_time: 3600 - name: List Python packages installed in virtualenv diff --git a/ansible/roles/kolla-bifrost/defaults/main.yml b/ansible/roles/kolla-bifrost/defaults/main.yml index 2d4ddb187..93dfc35c3 100644 --- a/ansible/roles/kolla-bifrost/defaults/main.yml +++ b/ansible/roles/kolla-bifrost/defaults/main.yml @@ -74,5 +74,17 @@ kolla_bifrost_ipa_ramdisk_checksum_algorithm: # Server inventory to be configured in {{ kolla_node_custom_config_path }}/bifrost/servers.yml. kolla_bifrost_servers: {} +# Deprecated. # Free form extra configuration to append to {{ kolla_node_custom_config_path }}/bifrost/bifrost.yml. kolla_bifrost_extra_globals: + +# Paths to Kolla Ansible custom configuration. +kolla_bifrost_config_paths_default: + - "{{ role_path }}/templates" + +# Paths to Kolla Ansible custom configuration. +kolla_bifrost_config_paths_extra: [] + +# Paths to Kolla Ansible custom configuration. Defaults to a combination of +# kolla_bifrost_config_paths_default and kolla_bifrost_config_paths_extra. +kolla_bifrost_config_paths: "{{ kolla_bifrost_config_paths_default + kolla_bifrost_config_paths_extra }}" diff --git a/ansible/roles/kolla-bifrost/tasks/main.yml b/ansible/roles/kolla-bifrost/tasks/main.yml index 3f199541f..a455048ff 100644 --- a/ansible/roles/kolla-bifrost/tasks/main.yml +++ b/ansible/roles/kolla-bifrost/tasks/main.yml @@ -6,11 +6,11 @@ mode: 0750 - name: Ensure the Kolla Bifrost configuration files exist - template: - src: "{{ item.src }}" - dest: "{{ kolla_node_custom_config_path }}/bifrost/{{ item.dest }}" + merge_yaml: + sources: "{{ kolla_bifrost_config_paths | product(['/kolla/config/bifrost/' ~ item]) | map('join') | list }}" + dest: "{{ kolla_node_custom_config_path }}/bifrost/{{ item }}" mode: 0640 with_items: - - { src: bifrost.yml.j2, dest: bifrost.yml } - - { src: dib.yml.j2, dest: dib.yml } - - { src: servers.yml.j2, dest: servers.yml } + - bifrost.yml + - dib.yml + - servers.yml diff --git a/ansible/roles/kolla-bifrost/templates/bifrost.yml.j2 b/ansible/roles/kolla-bifrost/templates/kolla/config/bifrost/bifrost.yml similarity index 100% rename from ansible/roles/kolla-bifrost/templates/bifrost.yml.j2 rename to ansible/roles/kolla-bifrost/templates/kolla/config/bifrost/bifrost.yml diff --git a/ansible/roles/kolla-bifrost/templates/dib.yml.j2 b/ansible/roles/kolla-bifrost/templates/kolla/config/bifrost/dib.yml similarity index 100% rename from ansible/roles/kolla-bifrost/templates/dib.yml.j2 rename to ansible/roles/kolla-bifrost/templates/kolla/config/bifrost/dib.yml diff --git a/ansible/roles/kolla-bifrost/templates/servers.yml.j2 b/ansible/roles/kolla-bifrost/templates/kolla/config/bifrost/servers.yml similarity index 100% rename from ansible/roles/kolla-bifrost/templates/servers.yml.j2 rename to ansible/roles/kolla-bifrost/templates/kolla/config/bifrost/servers.yml diff --git a/ansible/roles/kolla-build/defaults/main.yml b/ansible/roles/kolla-build/defaults/main.yml index 68ca8c38f..aecb0bfec 100644 --- a/ansible/roles/kolla-build/defaults/main.yml +++ b/ansible/roles/kolla-build/defaults/main.yml @@ -2,8 +2,15 @@ # Directory where Kolla config files will be installed. kolla_build_config_path: -# Path to extra kolla configuration files. -kolla_build_extra_config_path: +# Paths to extra kolla configuration files. +kolla_build_config_paths_default: + - "{{ role_path }}/templates/" + +# Paths to extra kolla configuration files. +kolla_build_config_paths_extra: [] + +# Paths to extra kolla configuration files. +kolla_build_config_paths: "{{ kolla_build_config_paths_default + kolla_build_config_paths_extra }}" # Valid options are [ centos, fedora, oraclelinux, ubuntu ] kolla_base_distro: diff --git a/ansible/roles/kolla-build/tasks/main.yml b/ansible/roles/kolla-build/tasks/main.yml index 08d6a74a2..82d5a2d43 100644 --- a/ansible/roles/kolla-build/tasks/main.yml +++ b/ansible/roles/kolla-build/tasks/main.yml @@ -1,23 +1,12 @@ --- -- name: Check whether a Kolla build extra configuration file exists - local_action: - module: stat - path: "{{ kolla_build_extra_config_path }}" - get_checksum: False - get_md5: False - mime: False - register: stat_result - -- name: Set a fact containing extra configuration - set_fact: - kolla_build_extra_config: "{{ lookup('template', kolla_build_extra_config_path) }}" - when: stat_result.stat.exists - -- name: Ensure the Kolla build configuration files exist - template: - src: "{{ item.src }}" - dest: "{{ kolla_build_config_path }}/{{ item.dest }}" +- name: Ensure the Kolla build configuration file exists + merge_configs: + sources: "{{ kolla_build_config_paths | product(['/kolla/kolla-build.conf']) | map('join') | list }}" + dest: "{{ kolla_build_config_path }}/kolla-build.conf" + mode: 0644 + +- name: Ensure the Kolla build template overrides file exists + template: + src: template-override.j2.j2 + dest: "{{ kolla_build_config_path }}/template-override.j2" mode: 0644 - with_items: - - { src: kolla-build.conf.j2, dest: kolla-build.conf } - - { src: template-override.j2.j2, dest: template-override.j2 } diff --git a/ansible/roles/kolla-build/templates/kolla-build.conf.j2 b/ansible/roles/kolla-build/templates/kolla/kolla-build.conf similarity index 97% rename from ansible/roles/kolla-build/templates/kolla-build.conf.j2 rename to ansible/roles/kolla-build/templates/kolla/kolla-build.conf index 5603bf787..78f5285d0 100644 --- a/ansible/roles/kolla-build/templates/kolla-build.conf.j2 +++ b/ansible/roles/kolla-build/templates/kolla/kolla-build.conf @@ -1,5 +1,3 @@ -# {{ ansible_managed }} - [DEFAULT] # Base container image distribution. diff --git a/doc/source/multiple-environments.rst b/doc/source/multiple-environments.rst index 1efc037f8..ecf89dab9 100644 --- a/doc/source/multiple-environments.rst +++ b/doc/source/multiple-environments.rst @@ -70,7 +70,7 @@ Ansible Inventories Each environment can include its own inventory, which overrides any variable declaration done in the shared inventory. Typically, a shared inventory may be used to define groups and group variables, while hosts and host variables would -be set in enviroment inventories. The following layout (ignoring non-inventory +be set in environment inventories. The following layout (ignoring non-inventory files) shows an example of multiple inventories. .. code-block:: text @@ -152,10 +152,31 @@ like the ``network-allocation.yml`` file. Kolla Configuration ------------------- -Kolla configuration is currently independent in each environment. To avoid -duplicating configuration, symbolic links can be used to share common variable -definitions. It is advised to avoid sharing credentials between environments by -making each Kolla ``passwords.yml`` file unique. +In the Wallaby release, Kolla configuration was independent in each +environment. + +As of the Xena release, the following files support combining the +environment-specific and shared configuration file content: + +* ``kolla/config/bifrost/bifrost.yml`` +* ``kolla/config/bifrost/dib.yml`` +* ``kolla/config/bifrost/servers.yml`` +* ``kolla/globals.yml`` +* ``kolla/kolla-build.conf`` + +Options in the environment-specific files take precedence over those in the +shared files. + +Managing Independent Environment Files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For files that are independent in each environment, i.e. they do not support +combining the environment-specific and shared configuration file content, there +are some techniques that may be used to avoid duplication. + +For example, symbolic links can be used to share common variable definitions. +It is advised to avoid sharing credentials between environments by making each +Kolla ``passwords.yml`` file unique. Custom Ansible Playbooks and Hooks ---------------------------------- diff --git a/kayobe/plugins/action/merge_yaml.py b/kayobe/plugins/action/merge_yaml.py index 72dc537be..b2e5a621a 100644 --- a/kayobe/plugins/action/merge_yaml.py +++ b/kayobe/plugins/action/merge_yaml.py @@ -86,7 +86,7 @@ class ActionModule(action.ActionBase): def read_config(self, source): result = None # Only use config if present - if os.access(source, os.R_OK): + if source and os.access(source, os.R_OK): with open(source, 'r') as f: template_data = f.read() diff --git a/releasenotes/notes/merge-configs-1f8fb3672e9be404.yaml b/releasenotes/notes/merge-configs-1f8fb3672e9be404.yaml new file mode 100644 index 000000000..e4892d520 --- /dev/null +++ b/releasenotes/notes/merge-configs-1f8fb3672e9be404.yaml @@ -0,0 +1,22 @@ +--- +features: + - | + Adds support for merging the following configuration files from the + environment-specific directory (``etc/kayobe/environments/``) + and the base directory (``etc/kayobe``). + + * ``kolla/config/bifrost/bifrost.yml`` + * ``kolla/config/bifrost/dib.yml`` + * ``kolla/config/bifrost/servers.yml`` + * ``kolla/globals.yml`` + * ``kolla/kolla-build.conf`` + + See `story 2002009 `__ + for details. +deprecations: + - | + The following variables are deprecated, in favour of using configuration + files ``kolla/globals.yml`` and ``kolla/kolla-build.conf`` respectively. + + * ``kolla_extra_globals`` + * ``kolla_bifrost_extra_globals`` diff --git a/tools/test-ansible.sh b/tools/test-ansible.sh index 93acd43e4..62f1904cf 100755 --- a/tools/test-ansible.sh +++ b/tools/test-ansible.sh @@ -6,6 +6,7 @@ set -e failed=0 +export ANSIBLE_ACTION_PLUGINS="kayobe/plugins/action:~/.ansible/plugins/action:/usr/share/ansible/plugins/action" for playbook in ansible/roles/*/tests/main.yml; do # We declare extra variables to install the {{ openstack_branch }} version # of kolla-ansible. We should use {{ kolla_ansible_source_version }}, but