Neutron upgrade testing.

Implement upgrade testing for os_neutron role.

This patch adds an upgrade testenv, which configures a previous version
of Neutron, and the test-vars from the previous stable branch to be used
by the previous version of Neutron.

We don't run functional tests after the initial stable/newton deploy
because a working stable/newton branch is already tested, and it's
assumed it worked if the upgrade works too - which is more
efficient.

This follows the pattern setup for Keystone in:
https://review.openstack.org/#/c/384269/

Change-Id: Id4b2cf8197ca902ab5b3d925175c3d7c698ec802
Implements: blueprint upgrade-testing

tests/ansible-role-requirements.yml

@@ -50,3 +50,7 @@
   scm: git
   src: https://github.com/logan2211/ansible-etcd
   version: master
+- name: os_previous_neutron
+  src: https://git.openstack.org/openstack/openstack-ansible-os_neutron
+  scm: git
+  version: stable/newton

tests/test-install-previous-neutron.yml

@@ -0,0 +1,94 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- include: common/setting-nodepool-variables.yml
+
+- name: Make /lib/modules accessible on neutron_agent containers
+  hosts: neutron_agent
+  user: root
+  gather_facts: true
+  tasks:
+    - name: Use the unconfined aa profile
+      lxc_container:
+        name: "{{ container_name }}"
+        container_config:
+          - "lxc.aa_profile=unconfined"
+      delegate_to: "{{ physical_host }}"
+
+    - name: Set mount path for kernel modules (Ubuntu)
+      set_fact:
+        kernel_module_path: "lib/modules"
+      when: ansible_pkg_mgr == 'apt'
+
+    - name: Set mount path for kernel modules (CentOS)
+      set_fact:
+        kernel_module_path: "usr/lib/modules"
+      when: ansible_pkg_mgr == 'yum'
+
+    - name: Neutron extra lxc config
+      lxc_container:
+        name: "{{ container_name }}"
+        container_command: |
+          [[ ! -d "/{{ kernel_module_path }}" ]] && mkdir -p "/{{ kernel_module_path }}"
+        container_config:
+          - "lxc.cgroup.devices.allow=a *:* rmw"
+          - "lxc.mount.entry=/{{ kernel_module_path }} {{ kernel_module_path }} none bind 0 0"
+      delegate_to: "{{ physical_host }}"
+
+    - name: Wait for ssh to be available
+      local_action:
+        module: wait_for
+        port: "{{ ansible_port | default('22') }}"
+        host: "{{ ansible_host | default(inventory_hostname) }}"
+        search_regex: OpenSSH
+        delay: 1
+
+    - name: Add iptables rule for communication w/ metadata agent
+      command: /sbin/iptables -t mangle -A POSTROUTING -p tcp --sport 80 -j CHECKSUM --checksum-fill
+
+- name: Deploy neutron
+  hosts: neutron_all
+  user: root
+  gather_facts: true
+  pre_tasks:
+    # NOTE: These are typically installed in the repo server where we build the
+    #       neutron wheel
+    - name: Install packages required to build neutron python package (Ubuntu)
+      apt:
+        name: "{{ item }}"
+      with_items:
+        - libffi-dev
+      when:
+        - inventory_hostname in groups['neutron_all']
+        - ansible_pkg_mgr == 'apt'
+    - name: Install packages required to build neutron python package (CentOS)
+      yum:
+        name: "{{ item }}"
+      with_items:
+        - libffi-devel
+      when:
+        - inventory_hostname in groups['neutron_all']
+        - ansible_pkg_mgr == 'yum'
+    - include: common/ensure-rabbitmq.yml
+      vhost_name: "{{ neutron_rabbitmq_vhost }}"
+      user_name: "{{ neutron_rabbitmq_userid }}"
+      user_password: "{{ neutron_rabbitmq_password }}"
+    - include: common/create-grant-db.yml
+      db_name: "{{ neutron_galera_database }}"
+      db_password: "{{ neutron_container_mysql_password }}"
+  roles:
+    - role: "os_previous_neutron"
+  vars_files:
+    - common/previous/test-vars.yml

tests/test-neutron-upgrades.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# WARNING:
+# This file is use by all OpenStack-Ansible roles for testing purposes.
+# Any changes here will affect all OpenStack-Ansible role repositories
+# with immediate effect.
+
+# PURPOSE:
+# This script executes test Ansible playbooks required for performing
+# an upgrade test of the os_neutron role.
+# Due to the way Ansible caches and handles modules, we need to run
+# separate Ansible runs to ensure the "upgrade" uses the new
+# "neutron_migrations_facts" module, instead of the cached version
+# used when deploying the previous Neutron version.
+
+## Shell Opts ----------------------------------------------------------------
+
+set -e
+
+## Vars ----------------------------------------------------------------------
+
+export WORKING_DIR=${WORKING_DIR:-$(pwd)}
+export ROLE_NAME=${ROLE_NAME:-''}
+
+export ANSIBLE_PARAMETERS=${ANSIBLE_PARAMETERS:-"-vvv"}
+export TEST_PLAYBOOK=${TEST_PLAYBOOK:-$WORKING_DIR/tests/test.yml}
+export TEST_CHECK_MODE=${TEST_CHECK_MODE:-false}
+export TEST_IDEMPOTENCE=${TEST_IDEMPOTENCE:-false}
+
+export COMMON_TESTS_PATH="${WORKING_DIR}/tests/common"
+
+echo "ANSIBLE_OVERRIDES: ${ANSIBLE_OVERRIDES}"
+echo "ANSIBLE_PARAMETERS: ${ANSIBLE_PARAMETERS}"
+echo "TEST_PLAYBOOK: ${TEST_PLAYBOOK}"
+echo "TEST_CHECK_MODE: ${TEST_CHECK_MODE}"
+echo "TEST_IDEMPOTENCE: ${TEST_IDEMPOTENCE}"
+
+## Functions -----------------------------------------------------------------
+
+function execute_ansible_playbook {
+
+  export ANSIBLE_CLI_PARAMETERS="${ANSIBLE_PARAMETERS}"
+  CMD_TO_EXECUTE="ansible-playbook ${TEST_PLAYBOOK} $@ ${ANSIBLE_CLI_PARAMETERS}"
+
+  echo "Executing: ${CMD_TO_EXECUTE}"
+  echo "With:"
+  echo "    ANSIBLE_INVENTORY: ${ANSIBLE_INVENTORY}"
+  echo "    ANSIBLE_LOG_PATH: ${ANSIBLE_LOG_PATH}"
+
+  ${CMD_TO_EXECUTE}
+
+}
+
+function gate_job_exit_tasks {
+  source "${COMMON_TESTS_PATH}/test-log-collect.sh"
+}
+
+## Main ----------------------------------------------------------------------
+
+# Ensure that the Ansible environment is properly prepared
+source "${COMMON_TESTS_PATH}/test-ansible-env-prep.sh"
+
+# Set gate job exit traps, this is run regardless of exit state when the job finishes.
+trap gate_job_exit_tasks EXIT
+
+# Prepare environment for the initial deploy of stable/newton Neutron
+# No upgrading or testing is done yet.
+export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-execute-newton-neutron-install.log"
+
+# Execute the setup of Stable/Newton Neutron
+execute_ansible_playbook
+
+# Prepare environment for the upgrade of Neutron
+export TEST_PLAYBOOK="${COMMON_TESTS_PATH}/test-install-neutron.yml"
+export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-execute-newton-upgrade.log"
+
+# Excute the upgrade of Neutron
+execute_ansible_playbook
+
+# Prepare the environment for the testing of upgraded Neutron
+export TEST_PLAYBOOK="${COMMON_TESTS_PATH}/test-install-tempest.yml"
+export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-execute-newton-upgrade-test.log"
+
+# Execute testing of upgraded Neutron
+execute_ansible_playbook

tests/test.yml

@@ -27,11 +27,21 @@
 # Install Keystone
 - include: common/test-install-keystone.yml
 
+# Install previous Neutron
+- include: test-install-previous-neutron.yml
+  when:
+    - neutron_upgrade is defined
+    - neutron_upgrade | bool
+
 # Install Neutron
 - include: common/test-install-neutron.yml
+  when:
+    - (neutron_upgrade is not defined) or (not neutron_upgrade | bool)
 
 # Install and execute tempest
 - include: common/test-install-tempest.yml
+  when:
+    - (neutron_upgrade is not defined) or (not neutron_upgrade | bool)
 
 - include: test-calico-functional.yml
   when:

tox.ini

@@ -110,6 +110,20 @@ commands =
     bash -c "{toxinidir}/tests/common/test-ansible-functional.sh"
 
 
+[testenv:upgrade]
+deps =
+    {[testenv:ansible]deps}
+setenv =
+    {[testenv]setenv}
+    ANSIBLE_PARAMETERS=-vvv -e neutron_upgrade=True
+commands =
+    {[testenv:tests_clone]commands}
+    bash -c "if [ ! -d "{toxinidir}/tests/common/previous" ]; then \
+               git clone https://git.openstack.org/openstack/openstack-ansible-tests -b stable/newton \
+                 {toxinidir}/tests/common/previous; \
+             fi"
+    bash -c "{toxinidir}/tests/test-neutron-upgrades.sh"
+
 [testenv:func_ovs]
 deps =
     {[testenv:ansible]deps}