Drop virtualenv pip package for CI

As CI images have pre-installed latest virtualenv package wich breaks
deployments. This will ensure that this package will be dropped and
appropriate one will be installed later.

This also makes ubuntu use appropriate python3 venv tooling and
drops py2 packages installation

Additionally virtualenv is dropped from requirements and ubuntu
upgrade jobs is marked as non-voting until backporting change
to previous release

Change-Id: I50931e0f68176bbaebfb68eec9f84241816c7fe3

requirements.txt

@@ -8,7 +8,6 @@ netaddr>=0.7.18 # BSD
 PrettyTable<0.8,>=0.7.1 # BSD
 python-memcached>=1.56 # PSF
 PyYAML>=3.12 # MIT
-virtualenv>=14.0.6 # MIT
 
 # We use this for our own keystone module.
 # TODO(odyssey4me): Remove this once we no

scripts/bootstrap-ansible.sh

@@ -88,11 +88,10 @@ case ${DISTRO_ID} in
         apt-get update
         DEBIAN_FRONTEND=noninteractive apt-get -y install \
           git-core curl gcc netcat \
-          python-minimal python-dev \
           python3 python3-dev \
           libssl-dev libffi-dev \
-          python-apt python3-apt \
-          python-virtualenv
+          python3-apt virtualenv \
+          python3-minimal
         ;;
     opensuse*)
         zypper -n install -l git-core curl autoconf gcc gcc-c++ \

scripts/gate-check-commit.sh

@@ -86,7 +86,9 @@ run_dstat || true
 load_nodepool_pip_opts
 
 # Bootstrap Ansible
-source "${OSA_CLONE_DIR}/scripts/bootstrap-ansible.sh"
+if [[ -z "${SKIP_OSA_BOOTSTRAP_AIO+defined}" ]]; then
+  source "${OSA_CLONE_DIR}/scripts/bootstrap-ansible.sh"
+fi
 
 # Flush all the iptables rules set by openstack-infra
 iptables -F

zuul.d/jobs.yaml

@@ -76,6 +76,7 @@
       - name: openstack/ansible-role-python_venv_build
       - name: openstack/ansible-role-uwsgi
     pre-run:
+      - zuul.d/playbooks/pre-gate-cleanup.yml
       - zuul.d/playbooks/pre-gate-scenario.yml
       - zuul.d/playbooks/pre-osa-aio.yml
     run: zuul.d/playbooks/run.yml

zuul.d/playbooks/pre-gate-cleanup.yml

@@ -0,0 +1,55 @@
+---
+# Copyright 2020, VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Cleanup gate images
+  hosts: all
+  become: yes
+  become_user: root
+  tasks:
+    - name: Gather variables for each operating system
+      include_vars: "{{ ansible_os_family | lower }}.yml"
+
+    - name: Remove known problem distro packages
+      package:
+        name: "{{ gate_packages_remove }}"
+        state: absent
+
+    - name: Remove known problem pip packages
+      pip:
+        name:
+          - appdirs
+          - backports
+          - contextlib2
+          - distlib
+          - filelock
+          - importlib_metadata
+          - importlib_resources
+          - six
+          - virtualenv
+          - zipp 
+        state: absent
+
+    - name: Remove package blacklist for yum/dnf
+      lineinfile:
+        dest: "{{ (ansible_pkg_mgr == 'dnf') | ternary('/etc/dnf/dnf.conf', '/etc/yum.conf') }}"
+        regexp: "^exclude="
+        state: absent
+      when: ansible_pkg_mgr in ['dnf', 'yum']
+
+    - name: Remove package blacklist for zypper
+      file:
+        path: /etc/zypp/locks
+        state: absent
+      when: ansible_pkg_mgr == 'zypper'

zuul.d/playbooks/vars/debian.yml

@@ -0,0 +1,19 @@
+---
+# Copyright 2020, VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+gate_packages_remove:
+  - python-virtualenv
+  - python3-virtualenv
+  - virtualenv

zuul.d/playbooks/vars/redhat.yml

@@ -0,0 +1,17 @@
+---
+# Copyright 2020, VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+gate_packages_remove:
+  - python-virtualenv

zuul.d/playbooks/vars/suse.yml

@@ -0,0 +1,18 @@
+---
+# Copyright 2020, VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+gate_packages_remove:
+  - python-virtualenv
+  - python2-virtualenv

zuul.d/project-templates.yaml

@@ -72,14 +72,14 @@
         - openstack-ansible-deploy-aio_metal-ubuntu-bionic
         - openstack-ansible-upgrade-aio_metal-centos-7:
             voting: false
-        - openstack-ansible-upgrade-aio_metal-ubuntu-bionic
+        - openstack-ansible-upgrade-aio_metal-ubuntu-bionic:
+            voting: false
     gate:
       jobs:
         - openstack-ansible-deploy-aio_metal-debian-buster
         - openstack-ansible-deploy-aio_metal-centos-7
         - openstack-ansible-deploy-aio_metal-opensuse-15
         - openstack-ansible-deploy-aio_metal-ubuntu-bionic
-        - openstack-ansible-upgrade-aio_metal-ubuntu-bionic
     periodic:
       jobs:
         - openstack-ansible-deploy-aio_metal-opensuse-15