CI: clean up requirements installation

We install kolla-ansible requirements in Zuul's Ansible playbooks.
This patch cleans up the installation in scripts so that they are
only concerned with auxiliary requirements:
- ansible (since we do not track it in requirements)
- ara (for log summaries)
- openstack clients (for first init and tests after deployment)

Additionally this patch installs openstack clients in a separate
virtualenv.
Note that all kolla-ansible requirements, ansible and ara are still
installed system-wide.

Change-Id: Iac04082ad39a9d823c515ba11c5db9af50ed225f
Signed-off-by: Radosław Piliszek <radoslaw.piliszek@gmail.com>

tests/deploy.sh

@@ -22,7 +22,10 @@ function deploy {
     tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy &> /tmp/logs/ansible/deploy
     tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy &> /tmp/logs/ansible/post-deploy
     tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check &> /tmp/logs/ansible/check-deploy
+
     . /etc/kolla/admin-openrc.sh
+    . ~/openstackclient-venv/bin/activate
+
     # Wait for service ready
     sleep 15
 

tests/test-ironic.sh

@@ -96,7 +96,7 @@ function create_resources {
 function test_ironic_logged {
     # Assumes init-runonce has been executed.
     . /etc/kolla/admin-openrc.sh
-    . ~/ironic-venv/bin/activate
+    . ~/openstackclient-venv/bin/activate
 
     # Smoke test ironic API.
     if ! openstack baremetal driver list | grep fake-hardware; then

tests/test-openstack.sh

@@ -8,6 +8,7 @@ export PYTHONUNBUFFERED=1
 
 function test_openstack_logged {
     . /etc/kolla/admin-openrc.sh
+    . ~/openstackclient-venv/bin/activate
 
     openstack --debug compute service list
     openstack --debug network agent list

tests/test-scenario-nfv.sh

@@ -50,11 +50,13 @@ function test_heat {
 
 function install_requirements {
     echo "TESTING: Install requirements"
-    sudo -H pip install --ignore-installed -U "python-tackerclient" "python-heatclient" "networking-sfc" "python-mistralclient" "python-barbicanclient"
+    pip install "python-tackerclient" "python-heatclient" "networking-sfc" "python-mistralclient" "python-barbicanclient"
 }
 
 function test_scenario_nfv_logged {
     . /etc/kolla/admin-openrc.sh
+    . ~/openstackclient-venv/bin/activate
+
     install_requirements
     test_tacker
     test_barbican

tools/setup_gate.sh

@@ -8,6 +8,19 @@ export PYTHONUNBUFFERED=1
 
 GIT_PROJECT_DIR=$(mktemp -d)
 
+function setup_openstack_clients {
+    # Prepare virtualenv for openstack deployment tests
+    virtualenv ~/openstackclient-venv
+    ~/openstackclient-venv/bin/pip install -U pip
+    ~/openstackclient-venv/bin/pip install python-openstackclient
+    if [[ $ACTION == zun ]]; then
+        ~/openstackclient-venv/bin/pip install python-zunclient
+    fi
+    if [[ $ACTION == ironic ]]; then
+        ~/openstackclient-venv/bin/pip install python-ironicclient
+    fi
+}
+
 function setup_config {
     # Use Infra provided pypi.
     # Wheel package mirror may be not compatible. So do not enable it.
@@ -98,18 +111,10 @@ function setup_ansible {
     else
         ANSIBLE_VERSION="<2.6"
     fi
+
     # TODO(SamYaple): Move to virtualenv
-    sudo -H pip install -U "ansible${ANSIBLE_VERSION}" "docker>=2.0.0" "python-openstackclient" "ara<1.0.0" "cmd2<0.9.0"
+    sudo pip install -U "ansible${ANSIBLE_VERSION}" "ara<1.0.0"
-    if [[ $ACTION == "zun" ]]; then
+
-        sudo -H pip install -U "python-zunclient"
-    fi
-    if [[ $ACTION == ironic ]]; then
-        # NOTE(mgoddard): Installing python-ironicclient to site-packages fails
-        # due to pip 10 distutils issue with ipaddress package.
-        virtualenv ~/ironic-venv
-        ~/ironic-venv/bin/pip install -U pip
-        ~/ironic-venv/bin/pip install python-openstackclient python-ironicclient
-    fi
     detect_distro
 
     sudo mkdir /etc/ansible
@@ -138,6 +143,8 @@ function prepare_images {
     popd
 }
 
+setup_openstack_clients
+
 setup_ansible
 setup_config
 setup_node