Update ansible-lint==4.3.5, flake8==3.8.3, bashate>=0.5.1

Correct lint errors that arise from these version updates.

This also fixes ceph deployment after backporting breaking changes
to stable-5.0 branch.

Change-Id: Iaa6bd137ebc4bf6cc2071712eb3f53a08c85d75e

.ansible-lint

@@ -0,0 +1,13 @@
+skip_list:
+  - '106'  # Role name {} does not match ``^[a-z][a-z0-9_]+$`` pattern
+  - '208'  # File permissions unset or incorrect
+
+warn_list:
+  - '204'  # Lines should be no longer than 160 chars
+  - '303'  # Using command rather than module
+  - '305'  # Use shell only when shell functionality is required
+  - '306'  # Shells that use pipes should set the pipefail option
+  - '502'  # All tasks should be named
+  - '602'  # Don't compare to empty string
+  - '701'  # meta/main.yml should contain relevant info
+  - '702'  # Tags must contain lowercase letters and digits only

inventory/group_vars/ceph_all.yml

@@ -18,6 +18,12 @@ ceph_container_bind_mounts:
   - bind_dir_path: "/var/lib/ceph"
     mount_path: "/openstack/{{ inventory_hostname }}"
 
+ceph_conf_overrides_log:
+  global:
+    log_to_syslog: true
+    err_to_syslog: true
+  mon:
+    mon_cluster_log_to_syslog: true
 
 ceph_conf_overrides_manila:
   client.manila:
@@ -26,7 +32,7 @@ ceph_conf_overrides_manila:
 # To extend ceph_conf_overrides use the ceph_conf_overrides_custom which will
 # combine any ceph_conf_overrides with other RGW specific overrides.
 ceph_conf_overrides_custom: {}
-ceph_conf_overrides: "{{ (ceph_conf_overrides_rgw | default({})) | combine(ceph_conf_overrides_manila, recursive=True) | combine(ceph_conf_overrides_custom, recursive=True) }}"
+ceph_conf_overrides: "{{ (ceph_conf_overrides_rgw | default({})) | combine(ceph_conf_overrides_manila, recursive=True) | combine(ceph_conf_overrides_custom, recursive=True) | combine(ceph_conf_overrides_log, recursive=True) }}"
 
 # Disable the NFS gateway PPA and package install by default as it is not
 # needed.
@@ -38,30 +44,4 @@ nfs_ganesha_stable: "{{ (ansible_distribution_release != 'bionic') }}"
 # ceph-ansible's default enabling of ntpd conflicts with the OSA defaults
 ntp_service_enabled: False
 
-
-openstack_cephfs_data_pool:
-  name: "manila_data"
-  pg_num: "{{ osd_pool_default_pg_num }}"
-  pgp_num: "{{ osd_pool_default_pg_num }}"
-  rule_name: "replicated_rule"
-  type: 1
-  erasure_profile: ""
-  expected_num_objects: ""
-  application: "cephfs"
-  size: "{{ osd_pool_default_size }}"
-  min_size: "{{ osd_pool_default_min_size }}"
-openstack_cephfs_metadata_pool:
-  name: "manila_metadata"
-  pg_num: "{{ osd_pool_default_pg_num }}"
-  pgp_num: "{{ osd_pool_default_pg_num }}"
-  rule_name: "replicated_rule"
-  type: 1
-  erasure_profile: ""
-  expected_num_objects: ""
-  application: "cephfs"
-  size: "{{ osd_pool_default_size }}"
-  min_size: "{{ osd_pool_default_min_size }}"
-
-cephfs_data_pool: "{{ openstack_cephfs_data_pool }}"
-cephfs_metadata_pool: "{{ openstack_cephfs_metadata_pool }}"
 dashboard_enabled: "{{ not (ansible_os_family | lower == 'redhat' and ansible_distribution_major_version is version('7', '=')) }}"

playbooks/ceph-install.yml

@@ -25,7 +25,6 @@
   gather_facts: false
   vars_files:
     - "defaults/{{ install_method }}_install.yml"
-    - "defaults/ceph_ansible.yml"
   pre_tasks:
 
     - import_tasks: common-tasks/ceph-server.yml
@@ -101,7 +100,6 @@
   gather_facts: false
   vars_files:
     - "defaults/{{ install_method }}_install.yml"
-    - "defaults/ceph_ansible.yml"
   pre_tasks:
 
     - import_tasks: common-tasks/ceph-server.yml
@@ -153,7 +151,6 @@
   user: root
   vars_files:
     - "defaults/{{ install_method }}_install.yml"
-    - "defaults/ceph_ansible.yml"
   roles:
     - role: ceph-defaults
       tags:

playbooks/ceph-nfs-install.yml

@@ -25,7 +25,6 @@
   user: root
   vars_files:
     - "defaults/{{ install_method }}_install.yml"
-    - "defaults/ceph_ansible.yml"
   pre_tasks:
     - name: Gather ceph-mon facts
       action: setup

playbooks/ceph-rgw-keystone-setup.yml

@@ -20,7 +20,8 @@
     - "defaults/{{ install_method }}_install.yml"
   vars:
     ansible_python_interpreter: >-
-      {{ openstack_service_setup_host_python_interpreter | default((openstack_service_setup_host | default('localhost') == 'localhost') | ternary(ansible_playbook_python, ansible_python['executable'])) }}
+      {{ openstack_service_setup_host_python_interpreter |
+         default((openstack_service_setup_host | default('localhost') == 'localhost') | ternary(ansible_playbook_python, ansible_python['executable'])) }}
   tags:
     - ceph-rgw
     - ceph-rgw-setup

playbooks/common-playbooks/cinder.yml

@@ -77,6 +77,7 @@
 
     - name: Add volume group block device to cinder
       shell: |
+        set -o pipefail
         {% if item.value.volume_group is defined %}
         if [ "$(pvdisplay | grep -B1 {{ item.value.volume_group }} | awk '/PV/ {print $3}')" ];then
           for device in `pvdisplay | grep -B1 {{ item.value.volume_group }} | awk '/PV/ {print $3}'`
@@ -86,6 +87,8 @@
         {% else %}
         echo "{{ item.key }} volume_group not defined"
         {% endif %}
+      args:
+        executable: /bin/bash
       with_dict: "{{ cinder_backends | default({}) }}"
       when:
         - container_tech | default('lxc') == 'lxc'

playbooks/common-tasks/ceph-server.yml

@@ -74,7 +74,9 @@
     - name: Install the EPEL repository
       yum_repository:
         name: epel-ceph
-        baseurl: "{{ centos_epel_mirror | default('http://download.fedoraproject.org/pub/epel') ~ '/' ~ ansible_distribution_major_version ~ '/' ~ ansible_architecture }}"
+        baseurl: "{{ centos_epel_mirror | default('http://download.fedoraproject.org/pub/epel') ~ '/'
+                   ~ ansible_distribution_major_version ~ '/'
+                   ~ ansible_architecture }}"
         description: 'Extra Packages for Enterprise Linux 7 - $basearch'
         gpgcheck: yes
         enabled: yes

playbooks/defaults/ceph_ansible.yml

@@ -1,21 +0,0 @@
----
-# Copyright 2019, VEXXHOST, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ceph_conf_overrides:
-  global:
-    log_to_syslog: true
-    err_to_syslog: true
-  mon:
-    mon_cluster_log_to_syslog: true

playbooks/openstack-hosts-setup.yml

@@ -60,8 +60,9 @@
             (ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7') or
             (ansible_os_family == 'RedHat' and ansible_distribution_major_version == '8') or
             (ansible_os_family == 'Suse' and ansible_distribution_major_version in ['15', '42'])
-        msg: "The only supported platforms for this release are Debian Stretch or Buster, Ubuntu 18.04 LTS (Bionic),  CentOS 7 (WIP) and openSUSE Leap 42.X and openSUSE Leap 15.X"
+        msg: >
-
+          The only supported platforms for this release are Debian Stretch or Buster,
+          Ubuntu 18.04 LTS (Bionic),  CentOS 7 (WIP) and openSUSE Leap 42.X and openSUSE Leap 15.X
   roles:
     - role: "openstack_hosts"
   vars_files:

playbooks/os-cinder-install.yml

@@ -68,7 +68,9 @@
     # venv tag for all hosts in the 'cinder_all' host group.
     - name: Gather software version list
       set_fact:
-        cinder_all_software_versions: "{{ (groups['cinder_all'] | map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'cinder', 'venv_tag'])) | list }}"
+        cinder_all_software_versions: "{{ (groups['cinder_all'] |
+                                           map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'cinder', 'venv_tag'])) |
+                                           list }}"
       delegate_to: localhost
       run_once: yes
 

playbooks/os-glance-install.yml

@@ -52,7 +52,9 @@
     # venv tag for all hosts in the 'glance_all' host group.
     - name: Gather software version list
       set_fact:
-        glance_all_software_versions: "{{ (groups['glance_all'] | map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'glance', 'venv_tag'])) | list }}"
+        glance_all_software_versions: "{{ (groups['glance_all'] |
+                                           map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'glance', 'venv_tag'])) |
+                                           list }}"
       delegate_to: localhost
       run_once: yes
 

playbooks/os-keystone-install.yml

@@ -107,7 +107,9 @@
     # venv tag for all hosts in the 'keystone_all' host group.
     - name: Gather software version list
       set_fact:
-        keystone_all_software_versions: "{{ (groups['keystone_all'] | map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'keystone', 'venv_tag'])) | list }}"
+        keystone_all_software_versions: "{{ (groups['keystone_all'] |
+                                             map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'keystone', 'venv_tag'])) |
+                                             list }}"
       delegate_to: localhost
       run_once: yes
 

playbooks/roles/system_crontab_coordination/meta/main.yml

@@ -23,6 +23,6 @@ galaxy_info:
   - name: Ubuntu
     versions:
     - bionic
-  categories:
+  galaxy_tags:
     - cloud
     - openstack

playbooks/utility-install.yml

@@ -28,7 +28,8 @@
     - "defaults/repo_packages/openstack_services.yml"
     - "defaults/{{ install_method }}_install.yml"
   vars:
-    utility_upper_constraints_url: "{{ requirements_git_url | default('https://releases.openstack.org/constraints/upper/' ~ requirements_git_install_branch | default('master')) }}"
+    utility_upper_constraints_url: "{{ requirements_git_url | default('https://releases.openstack.org/constraints/upper/' ~
+                                                                       requirements_git_install_branch | default('master')) }}"
   tags:
     - utility
   handlers:
@@ -72,7 +73,8 @@
   post_tasks:
     - name: Add OpenStack client to distro packages
       set_fact:
-        utility_distro_packages: "{{ (utility_distro_packages | default([])) + _utility_distro_openstack_clients_packages[utility_distro_openstack_clients_python_version|int] }}"
+        utility_distro_packages: "{{ (utility_distro_packages | default([])) +
+                                      _utility_distro_openstack_clients_packages[utility_distro_openstack_clients_python_version|int] }}"
       when: install_method == "distro"
 
     - name: Add EPEL repsitory
@@ -94,7 +96,9 @@
         - name: Install the EPEL repository
           yum_repository:
             name: epel-utility
-            baseurl: "{{ centos_epel_mirror | default('http://download.fedoraproject.org/pub/epel') ~ '/' ~ ansible_distribution_major_version ~ '/' ~ ansible_architecture }}"
+            baseurl: "{{ centos_epel_mirror | default('http://download.fedoraproject.org/pub/epel') ~ '/'
+                                                      ~ ansible_distribution_major_version ~ '/'
+                                                      ~ ansible_architecture }}"
             description: 'Extra Packages for Enterprise Linux 7 - $basearch'
             gpgcheck: yes
             enabled: yes

test-requirements.txt

@@ -11,6 +11,6 @@ virtualenv>=14.0.6 # MIT
 # TODO(noonedeadpunk) Upgrade linters versions and drop pycodestyle
 # once we stop doing linters check for the integrated repo with
 # openstack-ansible-tests/test-ansible-lint.sh
-flake8<2.6.0,>=2.5.4 # MIT
+flake8==3.8.3 # MIT
-ansible-lint==3.4.21 # MIT
+ansible-lint==4.3.5 # MIT
 pycodestyle>=2.5.0,<2.6.0

tests/bootstrap-aio.yml

@@ -81,8 +81,8 @@
     - name: Check that new network interfaces are up
       assert:
         that:
-          - ansible_eth12['active'] == true
+          - ansible_eth12['active'] | bool
-          - ansible_eth13['active'] == true
+          - ansible_eth13['active'] | bool
-          - ansible_eth14['active'] == true
+          - ansible_eth14['active'] | bool
       when:
         - (bootstrap_host_container_tech | default('unknown')) != 'nspawn'

tests/roles/bootstrap-host/tasks/check-requirements.yml

@@ -53,11 +53,14 @@
   # As such, use the btrfs tools to determine the real available size on the
   # disk
   shell: |
+    set -o pipefail
     if [[ $(df -T / | tail -n 1 | awk '{print $2}') == "btrfs" ]]; then
         btrfs fi usage --kbytes / | awk '/^.*Free / {print $3}'| sed 's/\..*//'
     else
         df -BK / | awk '!/^Filesystem/ {print $4}' | sed 's/K//'
     fi
+  args:
+    executable: /bin/bash
   environment:
     LC_ALL: en_US.UTF-8
   when:

tests/roles/bootstrap-host/tasks/detect_data_disk_device.yml

@@ -15,8 +15,11 @@
 
 - name: Locate data disk candidates
   shell: >
+    set -o pipefail;
     lsblk -brndo NAME,TYPE,RO,SIZE |
-    awk '/d[b-z]+ disk 0/{ if ($4>m && $4>={{ bootstrap_host_data_disk_min_size }}){m=$4; d=$1}}; END{print d}'
+    awk '/d[b-z]+ disk 0/{ if ($4>m && $4>={{ bootstrap_host_data_disk_min_size }}){m=$4; d=$1} }; END{print d}'
+  args:
+    executable: /bin/bash
   changed_when: false
   register: _data_disk
 
@@ -24,4 +27,4 @@
   set_fact:
     bootstrap_host_data_disk_device: "{{ _data_disk.stdout }}"
   when:
-    - _data_disk.stdout != ''
+    - _data_disk.stdout | length > 0

tests/roles/bootstrap-host/tasks/main.yml

@@ -65,7 +65,7 @@
     dest: '/openstack/src'
     state: link
   when:
-    - "lookup('env', 'ZUUL_SRC_PATH') != ''"
+    - "lookup('env', 'ZUUL_SRC_PATH') | length > 0"
 
 # Prepare the swap space loopback disk
 # This is only necessary if there isn't swap already

tests/roles/bootstrap-host/tasks/prepare_aio_config.yml

@@ -61,7 +61,10 @@
     - deploy-user-secrets
 
 - name: Generate any missing values in user_secrets
-  command: "/opt/ansible-runtime/bin/python {{ bootstrap_host_aio_script_path }}/pw-token-gen.py --file /etc/openstack_deploy/{{ bootstrap_host_user_secrets_filename }}"
+  command: >
+    /opt/ansible-runtime/bin/python
+    {{ bootstrap_host_aio_script_path }}/pw-token-gen.py
+    --file /etc/openstack_deploy/{{ bootstrap_host_user_secrets_filename }}
   changed_when: false
   tags:
     - generate_secrets

tests/roles/bootstrap-host/tasks/prepare_data_disk.yml

@@ -20,7 +20,10 @@
 #  in case these tasks are executed multiple times.
 - name: Determine whether partitions labeled openstack-data{1,2} are present
   shell: |
+    set -o pipefail;
     parted --script -l -m | egrep -q ':{{ bootstrap_host_data_disk_fs_type }}:openstack-data[12]:;$'
+  args:
+    executable: /bin/bash
   register: data_disk_partitions
   changed_when: false
   failed_when: false

tests/roles/bootstrap-host/templates/user_variables_ceph.yml.j2

@@ -19,7 +19,9 @@ monitor_interface: "{{ ('metal' in bootstrap_host_scenarios_expanded) | ternary(
 public_network: "{{ (mgmt_range ~ '.0/' ~ netmask) | ipaddr('net') }}"
 journal_size: 100
 osd_scenario: collocated
-pool_default_pg_num: 16
+ceph_conf_overrides_custom:
+  global:
+    mon_max_pg_per_osd: 500
 openstack_config: true # Ceph ansible automatically creates pools & keys
 cinder_ceph_client: cinder
 cinder_default_volume_type: RBD