Add metal upgrade jobs

This patch adds jobs for both CentOS 7 and Ubuntu upgrades to
run in check and gate to ensure that we don't break upgrade paths.

Depends-On: https://review.opendev.org/690072
Change-Id: I3609d68769075c8bbee77214f3790b89692dd0b0

scripts/gate-check-commit.sh

@@ -49,7 +49,7 @@ export INSTALL_METHOD=${3:-"source"}
 # Set the source branch for upgrade tests
 # Be sure to change this whenever a new stable branch
 # is created. The checkout must always be N-1.
-export UPGRADE_SOURCE_BRANCH=${UPGRADE_SOURCE_BRANCH:-'stable/rocky'}
+export UPGRADE_SOURCE_BRANCH=${UPGRADE_SOURCE_BRANCH:-'stable/stein'}
 
 # enable the ARA callback plugin
 export SETUP_ARA=${SETUP_ARA:-true}
@@ -63,9 +63,14 @@ export SETUP_ARA=${SETUP_ARA:-true}
 if [[ "${ACTION}" == "upgrade" ]]; then
     # Store the target SHA/branch
     export UPGRADE_TARGET_BRANCH=$(git rev-parse HEAD)
+    export OPENSTACK_SETUP_EXTRA_ARGS="-e tempest_install=no -e tempest_run=no"
 
     # Now checkout the source SHA/branch
     git checkout ${UPGRADE_SOURCE_BRANCH}
+
+    unset SKIP_OSA_RUNTIME_VENV_BUILD
+    unset SKIP_OSA_BOOTSTRAP_AIO
+    unset SKIP_OSA_ROLE_CLONE
 fi
 
 ## Functions -----------------------------------------------------------------
@@ -183,7 +188,7 @@ else
 
     # Setup OpenStack
     export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/setup-openstack.log"
-    openstack-ansible setup-openstack.yml -e osa_gather_facts=False
+    openstack-ansible setup-openstack.yml -e osa_gather_facts=False ${OPENSTACK_SETUP_EXTRA_ARGS:-}
 
     # Log some data about the instance and the rest of the system
     log_instance_info
@@ -209,7 +214,9 @@ if [[ "${ACTION}" == "upgrade" ]]; then
 
     # Source the current scripts-library.sh functions
     source "${OSA_CLONE_DIR}/scripts/scripts-library.sh"
-
+    # We need this as in stein we were deploying custom
+    # /etc/openstack_deploy/env.d/aio_metal.yml for metal installs
+    export SKIP_CUSTOM_ENVD_CHECK=true
     # To execute the upgrade script we need to provide
     # an affirmative response to the warning that the
     # upgrade is irreversable.

zuul.d/jobs.yaml

@@ -183,6 +183,12 @@
     nodeset: ubuntu-bionic
     timeout: 10800
 
+- job:
+    name: openstack-ansible-upgrade-aio_metal-ubuntu-bionic
+    parent: openstack-ansible-deploy-aio
+    nodeset: ubuntu-bionic
+    timeout: 10800
+
 - job:
     name: openstack-ansible-deploy-aio_nspawn-ubuntu-bionic
     parent: openstack-ansible-deploy-aio
@@ -256,6 +262,12 @@
     parent: openstack-ansible-deploy-aio
     nodeset: centos-7
 
+- job:
+    name: openstack-ansible-upgrade-aio_metal-centos-7
+    parent: openstack-ansible-deploy-aio
+    nodeset: centos-7
+    timeout: 10800
+
 - job:
     name: openstack-ansible-upgrade-aio_ceph-centos-7
     parent: openstack-ansible-deploy-aio

zuul.d/playbooks/pre-gate-scenario.yml

@@ -30,10 +30,10 @@
           {% set _scenario = [] %}
           {# Add the scenarios based on the job name #}
           {# ex. openstack-ansible-deploy-$scenario1_$scenario2-$os #}
-          {% if zuul.job is match('^openstack-ansible-deploy-([^-]+)-.*$') %}
+          {% if zuul.job is match('^openstack-ansible-(deploy|upgrade)-([^-]+)-.*$') %}
           {%   set _ = _scenario.extend(
                  (zuul.job |
-                  regex_replace('^openstack-ansible-deploy-([^-]+)-.*$', '\\1')).split('_')
+                  regex_replace('^openstack-ansible-[a-z]*-([^-]+)-.*$', '\\1')).split('_')
                )
           %}
           {#   If testing a role, add the role service name to the scenario list #}

zuul.d/playbooks/pre-osa-aio.yml

@@ -31,6 +31,7 @@
         ZUUL_SRC_PATH: "{{ ansible_user_dir }}/src"
         ANSIBLE_PACKAGE: "{{ ansible_package | default('') }}"
         SETUP_ARA: 'true'
+      when: action != 'upgrade'
     - name: Run bootstrap-aio script
       become: yes
       become_user: root
@@ -42,3 +43,4 @@
         SCENARIO: "{{ scenario }}"
         ACTION: "{{ action }}"
         INSTALL_METHOD: "{{ install_method }}"
+      when: action != 'upgrade'

zuul.d/project-templates.yaml

@@ -70,12 +70,16 @@
         - openstack-ansible-deploy-aio_metal-centos-7
         - openstack-ansible-deploy-aio_metal-opensuse-15
         - openstack-ansible-deploy-aio_metal-ubuntu-bionic
+        - openstack-ansible-upgrade-aio_metal-centos-7
+        - openstack-ansible-upgrade-aio_metal-ubuntu-bionic
     gate:
       jobs:
         - openstack-ansible-deploy-aio_metal-debian-stable
         - openstack-ansible-deploy-aio_metal-centos-7
         - openstack-ansible-deploy-aio_metal-opensuse-15
         - openstack-ansible-deploy-aio_metal-ubuntu-bionic
+        - openstack-ansible-upgrade-aio_metal-centos-7
+        - openstack-ansible-upgrade-aio_metal-ubuntu-bionic
     periodic:
       jobs:
         - openstack-ansible-deploy-aio_metal-opensuse-15