Add Centos-8 support

Depends-On: https://review.opendev.org/738903
Depends-On: https://review.opendev.org/735555
Depends-On: https://review.opendev.org/735781
Depends-On: https://review.opendev.org/735926
Depends-On: https://review.opendev.org/735971
Depends-On: https://review.opendev.org/736097
Depends-On: https://review.opendev.org/736095
Depends-On: https://review.opendev.org/736850
Depends-On: https://review.opendev.org/739026

Change-Id: Ibac278398ef533bc8fcf253c67cb983461bd145d

ansible-role-requirements.yml

@@ -245,7 +245,7 @@
   trackbranch: master
 - name: sshd
   scm: git
-  src: https://github.com/willshersystems/ansible-sshd
+  src: https://github.com/bengoa/ansible-sshd
   version: master
   trackbranch: master
 - name: bird

doc/source/user/aio/quickstart.rst

@@ -129,8 +129,8 @@ version.
 .. note::
    The |current_release_formal_name| release is only compatible with
    Debian 9 (stretch), Debian 10 (buster), Ubuntu 18.04 (Bionic Beaver),
-   CentOS 7. Experimentat support is provided for openSUSE Leap 15.X
+   CentOS 7 and CentOS 8. Experimentat support is provided for openSUSE
-   but this is expected to be removed in the next major release.
+   Leap 15.X but this is expected to be removed in the next major release.
 
 The next step is to bootstrap Ansible and the Ansible roles for the
 development environment.

inventory/group_vars/utility_all.yml

@@ -34,7 +34,7 @@ utility_distro_packages:
 # Packages installed into the utility container for distro installs
 # NOTE(jrosser) this can be simplified when all supported
 # distros use python3
-utility_distro_openstack_clients_python_version: "{{ (ansible_distribution == 'CentOS') | ternary('2', '3') }}"
+utility_distro_openstack_clients_python_version: "{{ (ansible_distribution == 'CentOS' and ansible_distribution_major_version is version('8', '<')) | ternary('2', '3') }}"
 
 _utility_distro_openstack_clients_packages:
   2:

playbooks/openstack-hosts-setup.yml

@@ -58,6 +58,7 @@
             (ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'bionic') or
             (ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'focal') or
             (ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7') or
+            (ansible_os_family == 'RedHat' and ansible_distribution_major_version == '8') or
             (ansible_os_family == 'Suse' and ansible_distribution_major_version in ['15', '42'])
         msg: "The only supported platforms for this release are Debian Stretch or Buster, Ubuntu 18.04 LTS (Bionic),  CentOS 7 (WIP) and openSUSE Leap 42.X and openSUSE Leap 15.X"
 

releasenotes/notes/add-centos-8-support-af546bd6ca3b10cd.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Support is added for deploying OpenStack on CentOS 8 with
+    source and distro based installs. However, nspawn support
+    can't be offered, as machinectl relies on btrfs which has
+    been dropped by CentOS.

scripts/bootstrap-ansible.sh

@@ -74,7 +74,7 @@ case ${DISTRO_ID} in
           git curl autoconf gcc gcc-c++ nc \
           python3 python3-devel libselinux-python3 \
           openssl-devel libffi-devel \
-          libselinux-python python-virtualenv
+          $(if [[ ${VERSION_ID} == '7' ]]; then echo "python-virtualenv"; else echo "python3-virtualenv"; fi)
         ;;
     ubuntu|debian)
         # NOTE(jrosser) remove this once infra debian images point to the upstream security repo

scripts/gate-check-commit.sh

@@ -91,15 +91,17 @@ if [[ -z "${SKIP_OSA_BOOTSTRAP_AIO+defined}" ]]; then
 fi
 
 # Flush all the iptables rules set by openstack-infra
-iptables -F
+if which iptables; then
-iptables -X
+  iptables -F
-iptables -t nat -F
+  iptables -X
-iptables -t nat -X
+  iptables -t nat -F
-iptables -t mangle -F
+  iptables -t nat -X
-iptables -t mangle -X
+  iptables -t mangle -F
-iptables -P INPUT ACCEPT
+  iptables -t mangle -X
-iptables -P FORWARD ACCEPT
+  iptables -P INPUT ACCEPT
-iptables -P OUTPUT ACCEPT
+  iptables -P FORWARD ACCEPT
+  iptables -P OUTPUT ACCEPT
+fi
 
 # Bootstrap an AIO
 if [[ -z "${SKIP_OSA_BOOTSTRAP_AIO+defined}" ]]; then

scripts/scripts-library.sh

@@ -87,7 +87,11 @@ function build_ansible_runtime_venv {
 
     # Add SELinux support to the venv
     if [ -d "/usr/lib64/python3.6/site-packages/selinux/" ]; then
-      rsync -avX /usr/lib64/python3.6/site-packages/selinux/ /opt/ansible-runtime/lib64/python3.6/selinux/
+      rsync -avX /usr/lib64/python3.6/site-packages/selinux/ /opt/ansible-runtime/lib64/python3.6/site-packages/selinux/
+    fi
+    # NOTE(noonedeadpunk) Conditional is here for compatability with CentOS 7
+    if [ -f "/usr/lib64/python3.6/site-packages/_selinux.cpython-36m-x86_64-linux-gnu.so" ]; then
+      rsync -avX /usr/lib64/python3.6/site-packages/_selinux.cpython-36m-x86_64-linux-gnu.so /opt/ansible-runtime/lib64/python3.6/site-packages/
     fi
 }
 
@@ -362,18 +366,6 @@ function get_instance_info {
           ;;
   esac
 
-  # Storage reports
-  for dir_name in lxc machines; do
-    btrfs filesystem usage /var/lib/${dir_name} > \
-      "/openstack/log/instance-info/btrfs_${dir_name}_usage_${TS}.log" || true
-    btrfs filesystem show /var/lib/${dir_name} > \
-      "/openstack/log/instance-info/btrfs_${dir_name}_show_${TS}.log" || true
-    btrfs filesystem df /var/lib/${dir_name} > \
-      "/openstack/log/instance-info/btrfs_${dir_name}_df_${TS}.log" || true
-    btrfs qgroup show --human-readable -pcre --iec /var/lib/${dir_name} > \
-      "/openstack/log/instance-info/btrfs_${dir_name}_quotas_${TS}.log" || true
-  done
-
   if command -v zfs >/dev/null; then
     zfs list > "/openstack/log/instance-info/zfs_lxc_${TS}.log" || true
   fi

tests/bootstrap-aio.yml

@@ -27,6 +27,7 @@
     sftp_subsystem:
         'apt': 'sftp /usr/lib/openssh/sftp-server'
         'yum': 'sftp /usr/libexec/openssh/sftp-server'
+        'dnf': 'sftp /usr/libexec/openssh/sftp-server'
         'zypper': 'sftp /usr/lib/ssh/sftp-server'
     sshd:
       ListenAddress:

tests/roles/bootstrap-host/defaults/main.yml

@@ -117,6 +117,10 @@ bootstrap_host_network_utils:
     iptables: /usr/sbin/iptables
     ethtool: /usr/sbin/ethtool
     ip: /usr/sbin/ip
+  dnf:
+    iptables: /usr/sbin/iptables
+    ethtool: /usr/sbin/ethtool
+    ip: /usr/sbin/ip
   zypper:
     iptables: /usr/sbin/iptables
     ethtool: /sbin/ethtool

tests/roles/bootstrap-host/tasks/check-requirements.yml

@@ -21,6 +21,7 @@
         (ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'bionic') or
         (ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'focal') or
         (ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7') or
+        (ansible_os_family == 'RedHat' and ansible_distribution_major_version == '8') or
         (ansible_os_family == 'Suse' and ansible_distribution_major_version in ['42', '15'])
     msg: >-
       The only supported platforms for this release are Debian Stretch, Ubuntu 18.04 LTS (Bionic),

tests/roles/bootstrap-host/tasks/prepare_aio_config.yml

@@ -85,20 +85,23 @@
         echo "centos_epel_mirror: '${NODEPOOL_EPEL_MIRROR}'" >> ${NODEPOOL_OVERRIDES}
         echo "galera_percona_xtrabackup_repo_host: '${NODEPOOL_PERCONA_PROXY}'" >> ${NODEPOOL_OVERRIDES}
         echo "galera_repo_host: '${NODEPOOL_MIRROR_HOST}:8080'" >> ${NODEPOOL_OVERRIDES}
-        echo "lxc_centos_package_baseurl: 'http://${NODEPOOL_MIRROR_HOST}:8080/copr-lxc2/epel-7-x86_64/'" >> ${NODEPOOL_OVERRIDES}
-        echo "lxc_centos_package_key: 'http://${NODEPOOL_MIRROR_HOST}:8080/copr-lxc2/pubkey.gpg'" >> ${NODEPOOL_OVERRIDES}
         echo "nova_virt_type: 'qemu'" >> ${NODEPOOL_OVERRIDES}
 
-        # NOTE(mnaser): We need to make sure we pull the latest RDO mirror
+        source /etc/os-release
-        #               which is hashed to avoid cache going stale during CI.
+        if [[ $ID == 'centos' && VERSION_ID == '7' ]]; then
-        export DLRN_BASE=${DLRN_BASE:-centos7-master/puppet-passed-ci}
+          echo "lxc_centos_package_baseurl: 'http://${NODEPOOL_MIRROR_HOST}:8080/copr-lxc2/epel-7-x86_64/'" >> ${NODEPOOL_OVERRIDES}
-        rdo_dlrn=`curl --silent ${NODEPOOL_RDO_PROXY}/${DLRN_BASE}/delorean.repo | grep baseurl | cut -d= -f2`
+          echo "lxc_centos_package_key: 'http://${NODEPOOL_MIRROR_HOST}:8080/copr-lxc2/pubkey.gpg'" >> ${NODEPOOL_OVERRIDES}
-        if [[ -z "$rdo_dlrn" ]]; then
+          # NOTE(mnaser): We need to make sure we pull the latest RDO mirror
-            echo "Failed to parse dlrn hash"
+          #               which is hashed to avoid cache going stale during CI.
-            exit 1
+          export DLRN_BASE=${DLRN_BASE:-centos7-master/puppet-passed-ci}
+          rdo_dlrn=`curl --silent ${NODEPOOL_RDO_PROXY}/${DLRN_BASE}/delorean.repo | grep baseurl | cut -d= -f2`
+          if [[ -z "$rdo_dlrn" ]]; then
+              echo "Failed to parse dlrn hash"
+              exit 1
+          fi
+          RDO_MIRROR_HOST=${rdo_dlrn/https:\/\/trunk.rdoproject.org/$NODEPOOL_RDO_PROXY}
+          echo "openstack_hosts_rdo_repo_url: '${RDO_MIRROR_HOST}'" >> ${NODEPOOL_OVERRIDES}
         fi
-        RDO_MIRROR_HOST=${rdo_dlrn/https:\/\/trunk.rdoproject.org/$NODEPOOL_RDO_PROXY}
-        echo "openstack_hosts_rdo_repo_url: '${RDO_MIRROR_HOST}'" >> ${NODEPOOL_OVERRIDES}
       args:
         executable: /bin/bash
       tags:

tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2

@@ -257,6 +257,8 @@ openstack_user_kernel_options:
   - key: 'kernel.printk'
     value: '4 1 7 4'
 
+openstack_hosts_package_state: latest
+
 {% if 'octavia' in bootstrap_host_scenarios_expanded %}
 # Enable Octavia V2 API/standalone
 octavia_v2: True

tests/roles/bootstrap-host/vars/redhat-8.yml

@@ -0,0 +1,35 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+packages_install:
+  # - bridge-utils  bridge-utils is not available in 8
+  # - btrfs-progs  BTRFS is not available in 8
+  - curl
+  - dbus
+  - ethtool
+  - git
+  - iputils
+  - lvm2
+  - python3
+  - python3-devel
+  # - sshpass  sshpass is not available in 8
+  #- systemd-networkd  networkd is not available in 8
+  - vim
+  - xfsprogs
+
+packages_remove: []
+
+rc_local: /etc/rc.d/rc.local
+rc_local_insert_before: "^touch /var/lock/subsys/local$"

zuul.d/jobs.yaml

@@ -344,6 +344,59 @@
     nodeset: centos-7
     timeout: 10800
 
+# centos 8
+- job:
+    name: openstack-ansible-deploy-aio_lxc-centos-8
+    parent: openstack-ansible-deploy-aio
+    nodeset: centos-8
+
+- job:
+    name: openstack-ansible-deploy-aio_distro_lxc-centos-8
+    parent: openstack-ansible-deploy-aio
+    nodeset: centos-8
+
+- job:
+    name: openstack-ansible-deploy-aio_ceph-centos-8
+    parent: openstack-ansible-deploy-aio
+    nodeset: centos-8
+
+- job:
+    name: openstack-ansible-deploy-aio_distro_ceph-centos-8
+    parent: openstack-ansible-deploy-aio
+    nodeset: centos-8
+
+- job:
+    name: openstack-ansible-upgrade-aio_lxc-centos-8
+    parent: openstack-ansible-deploy-aio
+    nodeset: centos-8
+    timeout: 10800
+
+- job:
+    name: openstack-ansible-deploy-aio_metal-centos-8
+    parent: openstack-ansible-deploy-aio
+    nodeset: centos-8
+
+- job:
+    name: openstack-ansible-deploy-aio_telemetry_metal-centos-8
+    parent: openstack-ansible-deploy-aio
+    nodeset: centos-8
+
+- job:
+    name: openstack-ansible-deploy-aio_distro_metal-centos-8
+    parent: openstack-ansible-deploy-aio
+    nodeset: centos-8
+
+- job:
+    name: openstack-ansible-deploy-aio_nspawn-centos-8
+    parent: openstack-ansible-deploy-aio
+    nodeset: centos-8
+
+- job:
+    name: openstack-ansible-upgrade-aio_ceph-centos-8
+    parent: openstack-ansible-deploy-aio
+    nodeset: centos-8
+    timeout: 10800
+
 # debian
 - job:
     name: openstack-ansible-deploy-aio_telemetry_metal-debian-buster

zuul.d/playbooks/pre-gate-cleanup.yml

@@ -30,6 +30,17 @@
           when:
             - ansible_distribution_release in ['bionic', 'focal']
 
+        # NOTE(noonedeadpunk): Added until CentOS 8.3 released because of the bug:
+        # https://bugzilla.redhat.com/show_bug.cgi?id=1777718
+        - name: Set umask to 022
+          lineinfile:
+            path: /etc/login.defs
+            regexp: "^UMASK"
+            line: "UMASK 022"
+          when:
+            - ansible_os_family | lower == 'redhat'
+            - ansible_distribution_major_version is version('8', '=')
+
     - name: Remove package blacklist for yum/dnf
       lineinfile:
         dest: "{{ (ansible_pkg_mgr == 'dnf') | ternary('/etc/dnf/dnf.conf', '/etc/yum.conf') }}"

zuul.d/project-templates.yaml

@@ -39,12 +39,14 @@
     check:
       jobs:
         - openstack-ansible-deploy-aio_lxc-centos-7
+        - openstack-ansible-deploy-aio_lxc-centos-8
         - openstack-ansible-deploy-aio_lxc-debian-buster
         - openstack-ansible-deploy-aio_lxc-ubuntu-bionic
         - openstack-ansible-deploy-aio_lxc-ubuntu-focal
     gate:
       jobs:
         - openstack-ansible-deploy-aio_lxc-centos-7
+        - openstack-ansible-deploy-aio_lxc-centos-8
         - openstack-ansible-deploy-aio_lxc-debian-buster
         - openstack-ansible-deploy-aio_lxc-ubuntu-bionic
         - openstack-ansible-deploy-aio_lxc-ubuntu-focal
@@ -61,6 +63,7 @@
     experimental:
       jobs:
         - openstack-ansible-deploy-aio_distro_lxc-centos-7
+        - openstack-ansible-deploy-aio_distro_lxc-centos-8
         - openstack-ansible-deploy-aio_distro_lxc-opensuse-15
         - openstack-ansible-deploy-aio_distro_lxc-ubuntu-bionic
 
@@ -70,6 +73,7 @@
       jobs:
         - openstack-ansible-deploy-aio_metal-debian-buster
         - openstack-ansible-deploy-aio_metal-centos-7
+        - openstack-ansible-deploy-aio_metal-centos-8
         - openstack-ansible-deploy-aio_metal-ubuntu-bionic
         - openstack-ansible-deploy-aio_metal-ubuntu-focal
         - openstack-ansible-upgrade-aio_metal-centos-7:
@@ -79,6 +83,7 @@
       jobs:
         - openstack-ansible-deploy-aio_metal-debian-buster
         - openstack-ansible-deploy-aio_metal-centos-7
+        - openstack-ansible-deploy-aio_lxc-centos-8
         - openstack-ansible-deploy-aio_metal-ubuntu-bionic
         - openstack-ansible-deploy-aio_metal-ubuntu-focal
     experimental:
@@ -106,10 +111,12 @@
     check:
       jobs:
         - openstack-ansible-deploy-aio_distro_metal-centos-7
+        - openstack-ansible-deploy-aio_distro_metal-centos-8
         - openstack-ansible-deploy-aio_distro_metal-ubuntu-bionic
     gate:
       jobs:
         - openstack-ansible-deploy-aio_distro_metal-centos-7
+        - openstack-ansible-deploy-aio_distro_metal-centos-8
         - openstack-ansible-deploy-aio_distro_metal-ubuntu-bionic
     experimental:
       jobs:

zuul.d/project.yaml

@@ -14,7 +14,7 @@
 
 - project:
     templates:
-      - openstack-ansible-deploy-guide-jobs
+      - deploy-guide-jobs
       - openstack-ansible-deploy-base-jobs
       - openstack-ansible-deploy-aio_lxc-jobs
       - openstack-ansible-deploy-aio_distro_lxc-jobs