Updated from OpenStack Ansible Tests

Change-Id: Id3d0a9ed55c653829abd1b7d37ace66b6167b478

.gitignore

@@ -61,12 +61,11 @@ ChangeLog
 releasenotes/build
 
 # Test temp files
-tests/plugins
-tests/playbooks
 tests/common
-tests/test.retry
+tests/*.retry
 
 # Vagrant artifacts
 .vagrant
 
-
+# Git clones
+openstack-ansible-ops

Vagrantfile

@@ -1,12 +1,35 @@
+# Verify whether required plugins are installed.
+required_plugins = [ "vagrant-disksize" ]
+required_plugins.each do |plugin|
+  if not Vagrant.has_plugin?(plugin)
+    raise "The vagrant plugin #{plugin} is required. Please run `vagrant plugin install #{plugin}`"
+  end
+end
+
 Vagrant.configure(2) do |config|
-  config.vm.box = "ubuntu/xenial64"
   config.vm.provider "virtualbox" do |v|
-    v.memory = 4096
+    v.memory = 2048
     v.cpus = 2
   end
-  config.vm.provision "shell", inline: <<-SHELL
+
-    sudo su -
+  config.vm.provision "shell",
+      privileged: false,
+      inline: <<-SHELL
           cd /vagrant
          ./run_tests.sh
       SHELL
+
+  config.vm.define "ubuntu1604" do |xenial|
+    xenial.disksize.size = "40GB"
+    xenial.vm.box = "ubuntu/xenial64"
+  end
+
+  config.vm.define "opensuse422" do |leap422|
+    leap422.vm.box = "opensuse/openSUSE-42.2-x86_64"
+  end
+
+  config.vm.define "centos7" do |centos7|
+    centos7.vm.box = "centos/7"
+  end
+
 end

bindep.txt

@@ -3,12 +3,24 @@
 #
 # See the following for details:
 #  - http://docs.openstack.org/infra/bindep/
-#  - https://github.com/openstack-infra/bindep
+#  - https://git.openstack.org/cgit/openstack-infra/bindep
 #
 # Even if the role does not make use of this facility, it
 # is better to have this file empty, otherwise OpenStack-CI
 # will fall back to installing its default packages which
 # will potentially be detrimental to the tests executed.
+#
+# Note:
+# This file is maintained in the openstack-ansible-tests repository.
+# https://git.openstack.org/cgit/openstack/openstack-ansible-tests/tree/bindep.txt
+# If you need to remove or add extra dependencies, you should modify
+# the central file instead and once your change is accepted then update
+# this file as well. The purpose of this file is to ensure that Python and
+# Ansible have all their necessary binary requirements on the test host before
+# tox executes. Any binary requirements needed by services/roles should be
+# installed by those roles in their applicable package install tasks, not through
+# using this file.
+#
 
 # Base requirements for Ubuntu
 build-essential   [platform:dpkg]
@@ -18,24 +30,27 @@ libffi-dev        [platform:dpkg]
 python2.7         [platform:dpkg]
 python-dev        [platform:dpkg]
 
-# Base requirements for CentOS
+# Base requirements for RPM distros
 gcc               [platform:rpm]
 gcc-c++           [platform:rpm]
 git               [platform:rpm]
-python-devel      [platform:rpm]
 libffi-devel      [platform:rpm]
 openssl-devel     [platform:rpm]
+python-devel      [platform:rpm]
 
 # For SELinux
-libselinux-python [platform:rpm]
+libselinux-python [platform:redhat]
+libsemanage-python [platform:redhat]
 
 # For SSL SNI support
-python-pyasn1               [platform:dpkg]
+python-pyasn1               [platform:dpkg platform:suse]
-python-openssl              [platform:dpkg]
+python-openssl              [platform:dpkg platform:suse]
 python-ndg-httpsclient      [platform:ubuntu]
-python2-pyasn1              [platform:rpm]
+python2-pyasn1              [platform:redhat]
-python2-pyOpenSSL           [platform:rpm]
+python2-pyOpenSSL           [platform:redhat]
-python-ndg_httpsclient      [platform:rpm]
+python-pyOpenSSL            [platform:suse]
+python-ndg_httpsclient      [platform:redhat]
+python-ndg-httpsclient      [platform:suse]
 
 # Required for compressing collected log files in CI
 gzip

run_tests.sh

@@ -12,63 +12,58 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+#
+# Note:
+# This file is maintained in the openstack-ansible-tests repository.
+# https://git.openstack.org/cgit/openstack/openstack-ansible-tests/tree/run_tests.sh
+# If you need to modify this file, update the one in the openstack-ansible-tests
+# repository and then update this file as well. The purpose of this file is to
+# prepare the host and then execute all the tox tests.
+#
 
-set -xeuo pipefail
+## Shell Opts ----------------------------------------------------------------
+set -xeu
 
-FUNCTIONAL_TEST=${FUNCTIONAL_TEST:-true}
+## Vars ----------------------------------------------------------------------
 
-# Install python2 for Ubuntu 16.04 and CentOS 7
+export WORKING_DIR=${WORKING_DIR:-$(pwd)}
-if which apt-get; then
-    sudo apt-get update && sudo apt-get install -y python
-fi
 
-if which yum; then
+## Main ----------------------------------------------------------------------
-    sudo yum install -y python
-fi
 
-# Install pip.
+source /etc/os-release || source /usr/lib/os-release
-if ! which pip; then
-  curl --silent --show-error --retry 5 \
-    https://bootstrap.pypa.io/get-pip.py | sudo python2.7
-fi
 
-# Install bindep and tox with pip.
+install_pkg_deps() {
-sudo pip install bindep tox
+    pkg_deps="git"
 
-# CentOS 7 requires two additional packages:
+    case ${ID,,} in
-#   redhat-lsb-core - for bindep profile support
+        *suse*) pkg_mgr_cmd="zypper -n in" ;;
-#   epel-release    - required to install python-ndg_httpsclient/python2-pyasn1
+        centos|rhel) pkg_mgr_cmd="yum install -y" ;;
-if which yum; then
+        fedora) pkg_mgr_cmd="dnf -y install" ;;
-    sudo yum -y install redhat-lsb-core epel-release
+        ubuntu|debian) pkg_mgr_cmd="apt-get install -y" ;;
-fi
+        *) echo "unsupported distribution: ${ID,,}"; exit 1 ;;
+    esac
 
-# Get a list of packages to install with bindep. If packages need to be
+    eval sudo $pkg_mgr_cmd $pkg_deps
-# installed, bindep exits with an exit code of 1.
+}
-BINDEP_PKGS=$(bindep -b -f bindep.txt test || true)
-echo "Packages to install: ${BINDEP_PKGS}"
 
-# Install a list of OS packages provided by bindep.
+git_clone_repo() {
-if which apt-get; then
+    if [[ ! -d tests/common ]]; then
-    sudo apt-get update
+        # The tests repo doesn't need a clone, we can just
-    DEBIAN_FRONTEND=noninteractive \
+        # symlink it.
-      sudo apt-get -q --option "Dpkg::Options::=--force-confold" \
+        if [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-tests" ]]; then
-      --assume-yes install $BINDEP_PKGS
+            ln -s ${WORKING_DIR} ${WORKING_DIR}/tests/common
-elif which yum; then
-    # Don't run yum with an empty list of packages.
-    # It will fail and cause the script to exit with an error.
-    if [[ ${#BINDEP_PKGS} > 0 ]]; then
-      sudo yum install -y $BINDEP_PKGS
-    fi
-fi
-
-# Loop through each tox environment and run tests.
-for tox_env in $(awk -F= '/envlist/ { gsub(",", " "); print $2 }' tox.ini); do
-  echo "Executing tox environment: ${tox_env}"
-  if [[ ${tox_env} == ansible-functional ]]; then
-    if ${FUNCTIONAL_TEST}; then
-      tox -e ${tox_env}
-    fi
         else
-    tox -e ${tox_env}
+            git clone \
+                https://git.openstack.org/openstack/openstack-ansible-tests \
+                tests/common
         fi
-done
+    fi
+}
+
+install_pkg_deps
+
+git_clone_repo
+
+# start executing the main test script
+source tests/common/run_tests_common.sh
+

tests/tests-repo-clone.sh

@@ -34,28 +34,39 @@ export TESTING_HOME=${TESTING_HOME:-$HOME}
 export WORKING_DIR=${WORKING_DIR:-$(pwd)}
 export CLONE_UPGRADE_TESTS=${CLONE_UPGRADE_TESTS:-no}
 
-## Main ----------------------------------------------------------------------
+## Functions -----------------------------------------------------------------
+
+function create_tests_clonemap {
 
 # Prepare the clonemap for zuul-cloner to use
-# This is placed here instead of inside the conditional
-# to prevent indentation problems.
 cat > ${TESTING_HOME}/tests-clonemap.yaml << EOF
 clonemap:
   - name: openstack/openstack-ansible-tests
     dest: ${WORKING_DIR}/tests/common
 EOF
 
+}
+
+## Main ----------------------------------------------------------------------
+
 # If zuul-cloner is present, use it so that we
 # also include any dependent patches from the
 # tests repo noted in the commit message.
 if [[ -x /usr/zuul-env/bin/zuul-cloner ]]; then
 
+    # Prepare the clonemap for zuul-cloner to use
+    create_tests_clonemap
+
+    # Execute the clone
     /usr/zuul-env/bin/zuul-cloner \
         --cache-dir /opt/git \
         --map ${TESTING_HOME}/tests-clonemap.yaml \
         git://git.openstack.org \
         openstack/openstack-ansible-tests
 
+    # Clean up the clonemap.
+    rm -f ${TESTING_HOME}/tests-clonemap.yaml
+
 # Alternatively, use a simple git-clone. We do
 # not re-clone if the directory exists already
 # to prevent overwriting any local changes which
@@ -73,9 +84,6 @@ elif [[ ! -d tests/common ]]; then
     fi
 fi
 
-# Clean up the clonemap.
-rm -f ${TESTING_HOME}/tests-clonemap.yaml
-
 # If this test set includes an upgrade test, the
 # previous stable release tests repo must also be
 # cloned.