From ec3233f9f8948bc3354121e5336ff5cbda0c41cb Mon Sep 17 00:00:00 2001 From: Jonathan Rosser Date: Tue, 29 Sep 2020 16:38:37 +0100 Subject: [PATCH] Update ansible-lint==4.3.5, flake8==3.8.3, bashate>=0.5.1 Correct lint errors that arise from these version updates. This also fixes ceph deployment after backporting breaking changes to stable-5.0 branch. Change-Id: Iaa6bd137ebc4bf6cc2071712eb3f53a08c85d75e --- .ansible-lint | 13 +++++++ inventory/group_vars/ceph_all.yml | 34 ++++--------------- playbooks/ceph-install.yml | 3 -- playbooks/ceph-nfs-install.yml | 1 - playbooks/ceph-rgw-keystone-setup.yml | 3 +- playbooks/common-playbooks/cinder.yml | 3 ++ playbooks/common-tasks/ceph-server.yml | 4 ++- playbooks/defaults/ceph_ansible.yml | 21 ------------ playbooks/openstack-hosts-setup.yml | 5 +-- playbooks/os-cinder-install.yml | 4 ++- playbooks/os-glance-install.yml | 4 ++- playbooks/os-keystone-install.yml | 4 ++- .../system_crontab_coordination/meta/main.yml | 2 +- playbooks/utility-install.yml | 10 ++++-- test-requirements.txt | 4 +-- tests/bootstrap-aio.yml | 6 ++-- .../tasks/check-requirements.yml | 3 ++ .../tasks/detect_data_disk_device.yml | 7 ++-- tests/roles/bootstrap-host/tasks/main.yml | 2 +- .../tasks/prepare_aio_config.yml | 5 ++- .../tasks/prepare_data_disk.yml | 3 ++ .../templates/user_variables_ceph.yml.j2 | 4 ++- 22 files changed, 72 insertions(+), 73 deletions(-) create mode 100644 .ansible-lint delete mode 100644 playbooks/defaults/ceph_ansible.yml diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000000..ffd4e9c0dd --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,13 @@ +skip_list: + - '106' # Role name {} does not match ``^[a-z][a-z0-9_]+$`` pattern + - '208' # File permissions unset or incorrect + +warn_list: + - '204' # Lines should be no longer than 160 chars + - '303' # Using command rather than module + - '305' # Use shell only when shell functionality is required + - '306' # Shells that use pipes should set the pipefail option + - '502' # All tasks should be named + - '602' # Don't compare to empty string + - '701' # meta/main.yml should contain relevant info + - '702' # Tags must contain lowercase letters and digits only \ No newline at end of file diff --git a/inventory/group_vars/ceph_all.yml b/inventory/group_vars/ceph_all.yml index 7d111c72eb..a790e4abf7 100644 --- a/inventory/group_vars/ceph_all.yml +++ b/inventory/group_vars/ceph_all.yml @@ -18,6 +18,12 @@ ceph_container_bind_mounts: - bind_dir_path: "/var/lib/ceph" mount_path: "/openstack/{{ inventory_hostname }}" +ceph_conf_overrides_log: + global: + log_to_syslog: true + err_to_syslog: true + mon: + mon_cluster_log_to_syslog: true ceph_conf_overrides_manila: client.manila: @@ -26,7 +32,7 @@ ceph_conf_overrides_manila: # To extend ceph_conf_overrides use the ceph_conf_overrides_custom which will # combine any ceph_conf_overrides with other RGW specific overrides. ceph_conf_overrides_custom: {} -ceph_conf_overrides: "{{ (ceph_conf_overrides_rgw | default({})) | combine(ceph_conf_overrides_manila, recursive=True) | combine(ceph_conf_overrides_custom, recursive=True) }}" +ceph_conf_overrides: "{{ (ceph_conf_overrides_rgw | default({})) | combine(ceph_conf_overrides_manila, recursive=True) | combine(ceph_conf_overrides_custom, recursive=True) | combine(ceph_conf_overrides_log, recursive=True) }}" # Disable the NFS gateway PPA and package install by default as it is not # needed. @@ -38,30 +44,4 @@ nfs_ganesha_stable: "{{ (ansible_distribution_release != 'bionic') }}" # ceph-ansible's default enabling of ntpd conflicts with the OSA defaults ntp_service_enabled: False - -openstack_cephfs_data_pool: - name: "manila_data" - pg_num: "{{ osd_pool_default_pg_num }}" - pgp_num: "{{ osd_pool_default_pg_num }}" - rule_name: "replicated_rule" - type: 1 - erasure_profile: "" - expected_num_objects: "" - application: "cephfs" - size: "{{ osd_pool_default_size }}" - min_size: "{{ osd_pool_default_min_size }}" -openstack_cephfs_metadata_pool: - name: "manila_metadata" - pg_num: "{{ osd_pool_default_pg_num }}" - pgp_num: "{{ osd_pool_default_pg_num }}" - rule_name: "replicated_rule" - type: 1 - erasure_profile: "" - expected_num_objects: "" - application: "cephfs" - size: "{{ osd_pool_default_size }}" - min_size: "{{ osd_pool_default_min_size }}" - -cephfs_data_pool: "{{ openstack_cephfs_data_pool }}" -cephfs_metadata_pool: "{{ openstack_cephfs_metadata_pool }}" dashboard_enabled: "{{ not (ansible_os_family | lower == 'redhat' and ansible_distribution_major_version is version('7', '=')) }}" diff --git a/playbooks/ceph-install.yml b/playbooks/ceph-install.yml index 58dfcc019b..3c72bc9d87 100644 --- a/playbooks/ceph-install.yml +++ b/playbooks/ceph-install.yml @@ -25,7 +25,6 @@ gather_facts: false vars_files: - "defaults/{{ install_method }}_install.yml" - - "defaults/ceph_ansible.yml" pre_tasks: - import_tasks: common-tasks/ceph-server.yml @@ -101,7 +100,6 @@ gather_facts: false vars_files: - "defaults/{{ install_method }}_install.yml" - - "defaults/ceph_ansible.yml" pre_tasks: - import_tasks: common-tasks/ceph-server.yml @@ -153,7 +151,6 @@ user: root vars_files: - "defaults/{{ install_method }}_install.yml" - - "defaults/ceph_ansible.yml" roles: - role: ceph-defaults tags: diff --git a/playbooks/ceph-nfs-install.yml b/playbooks/ceph-nfs-install.yml index d4e72b6079..e51b4e3fad 100644 --- a/playbooks/ceph-nfs-install.yml +++ b/playbooks/ceph-nfs-install.yml @@ -25,7 +25,6 @@ user: root vars_files: - "defaults/{{ install_method }}_install.yml" - - "defaults/ceph_ansible.yml" pre_tasks: - name: Gather ceph-mon facts action: setup diff --git a/playbooks/ceph-rgw-keystone-setup.yml b/playbooks/ceph-rgw-keystone-setup.yml index e0def2f0b1..06b2404e5e 100644 --- a/playbooks/ceph-rgw-keystone-setup.yml +++ b/playbooks/ceph-rgw-keystone-setup.yml @@ -20,7 +20,8 @@ - "defaults/{{ install_method }}_install.yml" vars: ansible_python_interpreter: >- - {{ openstack_service_setup_host_python_interpreter | default((openstack_service_setup_host | default('localhost') == 'localhost') | ternary(ansible_playbook_python, ansible_python['executable'])) }} + {{ openstack_service_setup_host_python_interpreter | + default((openstack_service_setup_host | default('localhost') == 'localhost') | ternary(ansible_playbook_python, ansible_python['executable'])) }} tags: - ceph-rgw - ceph-rgw-setup diff --git a/playbooks/common-playbooks/cinder.yml b/playbooks/common-playbooks/cinder.yml index 0ebc011d34..72d1b95595 100644 --- a/playbooks/common-playbooks/cinder.yml +++ b/playbooks/common-playbooks/cinder.yml @@ -77,6 +77,7 @@ - name: Add volume group block device to cinder shell: | + set -o pipefail {% if item.value.volume_group is defined %} if [ "$(pvdisplay | grep -B1 {{ item.value.volume_group }} | awk '/PV/ {print $3}')" ];then for device in `pvdisplay | grep -B1 {{ item.value.volume_group }} | awk '/PV/ {print $3}'` @@ -86,6 +87,8 @@ {% else %} echo "{{ item.key }} volume_group not defined" {% endif %} + args: + executable: /bin/bash with_dict: "{{ cinder_backends | default({}) }}" when: - container_tech | default('lxc') == 'lxc' diff --git a/playbooks/common-tasks/ceph-server.yml b/playbooks/common-tasks/ceph-server.yml index e0fd653fbd..d2c28349c4 100644 --- a/playbooks/common-tasks/ceph-server.yml +++ b/playbooks/common-tasks/ceph-server.yml @@ -74,7 +74,9 @@ - name: Install the EPEL repository yum_repository: name: epel-ceph - baseurl: "{{ centos_epel_mirror | default('http://download.fedoraproject.org/pub/epel') ~ '/' ~ ansible_distribution_major_version ~ '/' ~ ansible_architecture }}" + baseurl: "{{ centos_epel_mirror | default('http://download.fedoraproject.org/pub/epel') ~ '/' + ~ ansible_distribution_major_version ~ '/' + ~ ansible_architecture }}" description: 'Extra Packages for Enterprise Linux 7 - $basearch' gpgcheck: yes enabled: yes diff --git a/playbooks/defaults/ceph_ansible.yml b/playbooks/defaults/ceph_ansible.yml deleted file mode 100644 index 381293e76b..0000000000 --- a/playbooks/defaults/ceph_ansible.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -# Copyright 2019, VEXXHOST, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ceph_conf_overrides: - global: - log_to_syslog: true - err_to_syslog: true - mon: - mon_cluster_log_to_syslog: true diff --git a/playbooks/openstack-hosts-setup.yml b/playbooks/openstack-hosts-setup.yml index e057acb1e3..c092b52d78 100644 --- a/playbooks/openstack-hosts-setup.yml +++ b/playbooks/openstack-hosts-setup.yml @@ -60,8 +60,9 @@ (ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7') or (ansible_os_family == 'RedHat' and ansible_distribution_major_version == '8') or (ansible_os_family == 'Suse' and ansible_distribution_major_version in ['15', '42']) - msg: "The only supported platforms for this release are Debian Stretch or Buster, Ubuntu 18.04 LTS (Bionic), CentOS 7 (WIP) and openSUSE Leap 42.X and openSUSE Leap 15.X" - + msg: > + The only supported platforms for this release are Debian Stretch or Buster, + Ubuntu 18.04 LTS (Bionic), CentOS 7 (WIP) and openSUSE Leap 42.X and openSUSE Leap 15.X roles: - role: "openstack_hosts" vars_files: diff --git a/playbooks/os-cinder-install.yml b/playbooks/os-cinder-install.yml index 283f43749f..05e99f20c8 100644 --- a/playbooks/os-cinder-install.yml +++ b/playbooks/os-cinder-install.yml @@ -68,7 +68,9 @@ # venv tag for all hosts in the 'cinder_all' host group. - name: Gather software version list set_fact: - cinder_all_software_versions: "{{ (groups['cinder_all'] | map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'cinder', 'venv_tag'])) | list }}" + cinder_all_software_versions: "{{ (groups['cinder_all'] | + map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'cinder', 'venv_tag'])) | + list }}" delegate_to: localhost run_once: yes diff --git a/playbooks/os-glance-install.yml b/playbooks/os-glance-install.yml index 2c9d58758b..fb4a23b1ba 100644 --- a/playbooks/os-glance-install.yml +++ b/playbooks/os-glance-install.yml @@ -52,7 +52,9 @@ # venv tag for all hosts in the 'glance_all' host group. - name: Gather software version list set_fact: - glance_all_software_versions: "{{ (groups['glance_all'] | map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'glance', 'venv_tag'])) | list }}" + glance_all_software_versions: "{{ (groups['glance_all'] | + map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'glance', 'venv_tag'])) | + list }}" delegate_to: localhost run_once: yes diff --git a/playbooks/os-keystone-install.yml b/playbooks/os-keystone-install.yml index 12ea16d549..1b09019b7a 100644 --- a/playbooks/os-keystone-install.yml +++ b/playbooks/os-keystone-install.yml @@ -107,7 +107,9 @@ # venv tag for all hosts in the 'keystone_all' host group. - name: Gather software version list set_fact: - keystone_all_software_versions: "{{ (groups['keystone_all'] | map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'keystone', 'venv_tag'])) | list }}" + keystone_all_software_versions: "{{ (groups['keystone_all'] | + map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'keystone', 'venv_tag'])) | + list }}" delegate_to: localhost run_once: yes diff --git a/playbooks/roles/system_crontab_coordination/meta/main.yml b/playbooks/roles/system_crontab_coordination/meta/main.yml index f9be3fbb56..ae07d55e6e 100644 --- a/playbooks/roles/system_crontab_coordination/meta/main.yml +++ b/playbooks/roles/system_crontab_coordination/meta/main.yml @@ -23,6 +23,6 @@ galaxy_info: - name: Ubuntu versions: - bionic - categories: + galaxy_tags: - cloud - openstack diff --git a/playbooks/utility-install.yml b/playbooks/utility-install.yml index 106dcaa81e..84bb21c7a9 100644 --- a/playbooks/utility-install.yml +++ b/playbooks/utility-install.yml @@ -28,7 +28,8 @@ - "defaults/repo_packages/openstack_services.yml" - "defaults/{{ install_method }}_install.yml" vars: - utility_upper_constraints_url: "{{ requirements_git_url | default('https://releases.openstack.org/constraints/upper/' ~ requirements_git_install_branch | default('master')) }}" + utility_upper_constraints_url: "{{ requirements_git_url | default('https://releases.openstack.org/constraints/upper/' ~ + requirements_git_install_branch | default('master')) }}" tags: - utility handlers: @@ -72,7 +73,8 @@ post_tasks: - name: Add OpenStack client to distro packages set_fact: - utility_distro_packages: "{{ (utility_distro_packages | default([])) + _utility_distro_openstack_clients_packages[utility_distro_openstack_clients_python_version|int] }}" + utility_distro_packages: "{{ (utility_distro_packages | default([])) + + _utility_distro_openstack_clients_packages[utility_distro_openstack_clients_python_version|int] }}" when: install_method == "distro" - name: Add EPEL repsitory @@ -94,7 +96,9 @@ - name: Install the EPEL repository yum_repository: name: epel-utility - baseurl: "{{ centos_epel_mirror | default('http://download.fedoraproject.org/pub/epel') ~ '/' ~ ansible_distribution_major_version ~ '/' ~ ansible_architecture }}" + baseurl: "{{ centos_epel_mirror | default('http://download.fedoraproject.org/pub/epel') ~ '/' + ~ ansible_distribution_major_version ~ '/' + ~ ansible_architecture }}" description: 'Extra Packages for Enterprise Linux 7 - $basearch' gpgcheck: yes enabled: yes diff --git a/test-requirements.txt b/test-requirements.txt index fdc28eb468..c3ed8ab81b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,6 +11,6 @@ virtualenv>=14.0.6 # MIT # TODO(noonedeadpunk) Upgrade linters versions and drop pycodestyle # once we stop doing linters check for the integrated repo with # openstack-ansible-tests/test-ansible-lint.sh -flake8<2.6.0,>=2.5.4 # MIT -ansible-lint==3.4.21 # MIT +flake8==3.8.3 # MIT +ansible-lint==4.3.5 # MIT pycodestyle>=2.5.0,<2.6.0 diff --git a/tests/bootstrap-aio.yml b/tests/bootstrap-aio.yml index def4efe633..8a87660f8e 100644 --- a/tests/bootstrap-aio.yml +++ b/tests/bootstrap-aio.yml @@ -81,8 +81,8 @@ - name: Check that new network interfaces are up assert: that: - - ansible_eth12['active'] == true - - ansible_eth13['active'] == true - - ansible_eth14['active'] == true + - ansible_eth12['active'] | bool + - ansible_eth13['active'] | bool + - ansible_eth14['active'] | bool when: - (bootstrap_host_container_tech | default('unknown')) != 'nspawn' diff --git a/tests/roles/bootstrap-host/tasks/check-requirements.yml b/tests/roles/bootstrap-host/tasks/check-requirements.yml index 914b64c68b..1aabca17e0 100644 --- a/tests/roles/bootstrap-host/tasks/check-requirements.yml +++ b/tests/roles/bootstrap-host/tasks/check-requirements.yml @@ -53,11 +53,14 @@ # As such, use the btrfs tools to determine the real available size on the # disk shell: | + set -o pipefail if [[ $(df -T / | tail -n 1 | awk '{print $2}') == "btrfs" ]]; then btrfs fi usage --kbytes / | awk '/^.*Free / {print $3}'| sed 's/\..*//' else df -BK / | awk '!/^Filesystem/ {print $4}' | sed 's/K//' fi + args: + executable: /bin/bash environment: LC_ALL: en_US.UTF-8 when: diff --git a/tests/roles/bootstrap-host/tasks/detect_data_disk_device.yml b/tests/roles/bootstrap-host/tasks/detect_data_disk_device.yml index 7110c161f2..ecfef28ca3 100644 --- a/tests/roles/bootstrap-host/tasks/detect_data_disk_device.yml +++ b/tests/roles/bootstrap-host/tasks/detect_data_disk_device.yml @@ -15,8 +15,11 @@ - name: Locate data disk candidates shell: > + set -o pipefail; lsblk -brndo NAME,TYPE,RO,SIZE | - awk '/d[b-z]+ disk 0/{ if ($4>m && $4>={{ bootstrap_host_data_disk_min_size }}){m=$4; d=$1}}; END{print d}' + awk '/d[b-z]+ disk 0/{ if ($4>m && $4>={{ bootstrap_host_data_disk_min_size }}){m=$4; d=$1} }; END{print d}' + args: + executable: /bin/bash changed_when: false register: _data_disk @@ -24,4 +27,4 @@ set_fact: bootstrap_host_data_disk_device: "{{ _data_disk.stdout }}" when: - - _data_disk.stdout != '' + - _data_disk.stdout | length > 0 diff --git a/tests/roles/bootstrap-host/tasks/main.yml b/tests/roles/bootstrap-host/tasks/main.yml index cab28153d6..40bf0d6a27 100644 --- a/tests/roles/bootstrap-host/tasks/main.yml +++ b/tests/roles/bootstrap-host/tasks/main.yml @@ -65,7 +65,7 @@ dest: '/openstack/src' state: link when: - - "lookup('env', 'ZUUL_SRC_PATH') != ''" + - "lookup('env', 'ZUUL_SRC_PATH') | length > 0" # Prepare the swap space loopback disk # This is only necessary if there isn't swap already diff --git a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml index bb9246b7f8..ab9c8eb16c 100644 --- a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml +++ b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml @@ -61,7 +61,10 @@ - deploy-user-secrets - name: Generate any missing values in user_secrets - command: "/opt/ansible-runtime/bin/python {{ bootstrap_host_aio_script_path }}/pw-token-gen.py --file /etc/openstack_deploy/{{ bootstrap_host_user_secrets_filename }}" + command: > + /opt/ansible-runtime/bin/python + {{ bootstrap_host_aio_script_path }}/pw-token-gen.py + --file /etc/openstack_deploy/{{ bootstrap_host_user_secrets_filename }} changed_when: false tags: - generate_secrets diff --git a/tests/roles/bootstrap-host/tasks/prepare_data_disk.yml b/tests/roles/bootstrap-host/tasks/prepare_data_disk.yml index 1012828ae2..2ec1b2b726 100644 --- a/tests/roles/bootstrap-host/tasks/prepare_data_disk.yml +++ b/tests/roles/bootstrap-host/tasks/prepare_data_disk.yml @@ -20,7 +20,10 @@ # in case these tasks are executed multiple times. - name: Determine whether partitions labeled openstack-data{1,2} are present shell: | + set -o pipefail; parted --script -l -m | egrep -q ':{{ bootstrap_host_data_disk_fs_type }}:openstack-data[12]:;$' + args: + executable: /bin/bash register: data_disk_partitions changed_when: false failed_when: false diff --git a/tests/roles/bootstrap-host/templates/user_variables_ceph.yml.j2 b/tests/roles/bootstrap-host/templates/user_variables_ceph.yml.j2 index 66fd2e6c9c..edc7eb4913 100644 --- a/tests/roles/bootstrap-host/templates/user_variables_ceph.yml.j2 +++ b/tests/roles/bootstrap-host/templates/user_variables_ceph.yml.j2 @@ -19,7 +19,9 @@ monitor_interface: "{{ ('metal' in bootstrap_host_scenarios_expanded) | ternary( public_network: "{{ (mgmt_range ~ '.0/' ~ netmask) | ipaddr('net') }}" journal_size: 100 osd_scenario: collocated -pool_default_pg_num: 16 +ceph_conf_overrides_custom: + global: + mon_max_pg_per_osd: 500 openstack_config: true # Ceph ansible automatically creates pools & keys cinder_ceph_client: cinder cinder_default_volume_type: RBD