Neutron test cleanup

This commit does the following:

- fully defines the inventory rather than having test-prepare-host.yml
  add containers to it
- updates test-prepare-keys.yml to use localhost instead of 127.0.0.1,
  uses become_user=jenkins instead of become=false, and removes
  redundant connection
- updates test-prepare-host.yml by changing 127.0.0.1 to localhost, and
  removes mocking ansible_env
- groups vars in defined in each individual file into a single
  test-vars.yml file
- installs libffi-dev in test-install-neutron.yml, which is necessary
  to build neutron (installed by default in gate, so not an issue
  there)
- switches neutron_provider_networks to vxlan, otherwise
  neutron-linuxbridge-agent doesn't actually work correctly

Change-Id: Ice7e4762fb184d9172c6b8192f4e6d1732654bb9

tests/group_vars/all_containers.yml

@@ -0,0 +1,27 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ansible_ssh_host: "{{ ansible_host }}"
+container_name: "{{ inventory_hostname }}"
+container_networks:
+  management_address:
+    address: "{{ ansible_host }}"
+    bridge: "lxcbr0"
+    interface: "eth1"
+    netmask: "255.255.252.0"
+    type: "veth"
+physical_host: localhost
+properties:
+  service_name: "{{ inventory_hostname }}"

tests/inventory

@@ -1,2 +1,55 @@
 [all]
 localhost ansible_connection=local ansible_become=True
+infra1 ansible_host=10.100.100.2 ansible_become=True ansible_user=root
+openstack1 ansible_host=10.100.100.3 ansible_become=True ansible_user=root
+
+[all_containers]
+infra1
+openstack1
+
+[rabbitmq_all]
+infra1
+
+[galera_all]
+infra1
+
+[service_all:children]
+rabbitmq_all
+galera_all
+
+[keystone_all]
+openstack1
+
+[neutron_agent]
+openstack1
+
+[neutron_dhcp_agent]
+openstack1
+
+[neutron_linuxbridge_agent]
+openstack1
+
+[neutron_metering_agent]
+openstack1
+
+[neutron_l3_agent]
+openstack1
+
+[neutron_lbaas_agent]
+openstack1
+
+[neutron_metadata_agent]
+openstack1
+
+[neutron_server]
+openstack1
+
+[neutron_all:children]
+neutron_agent
+neutron_dhcp_agent
+neutron_linuxbridge_agent
+neutron_metering_agent
+neutron_l3_agent
+neutron_lbaas_agent
+neutron_metadata_agent
+neutron_server

tests/test-install-keystone.yml

@@ -18,6 +18,14 @@
   user: root
   gather_facts: true
   pre_tasks:
+    # NOTE: These are typically installed in the repo server where we build the
+    #       neutron wheel
+    - name: Install packages required to build neutron python package
+      apt:
+        name: "{{ item }}"
+      with_items:
+        - libffi-dev
+      when: inventory_hostname in groups['neutron_all']
     - name: Ensure rabbitmq vhost
       rabbitmq_vhost:
         name: "{{ keystone_rabbitmq_vhost }}"
@@ -71,23 +79,5 @@
         - mysql-db-setup
   roles:
     - role: os_keystone
-  vars:
+  vars_files:
-    external_lb_vip_address: 10.100.100.3
+    - test-vars.yml
-    internal_lb_vip_address: 10.100.100.3
-    keystone_galera_address: 10.100.100.2
-    keystone_galera_database: keystone
-    keystone_venv_tag: testing
-    keystone_developer_mode: true
-    keystone_git_install_branch: stable/mitaka
-    keystone_requirements_git_install_branch: stable/mitaka
-    keystone_auth_admin_token: SuperSecreteTestToken
-    keystone_auth_admin_password: SuperSecretePassword
-    keystone_service_password: secrete
-    keystone_rabbitmq_password: secrete
-    keystone_container_mysql_password: SuperSecrete
-    keystone_rabbitmq_port: 5671
-    keystone_rabbitmq_userid: keystone
-    keystone_rabbitmq_vhost: /keystone
-    keystone_rabbitmq_servers: 10.100.100.2
-    keystone_rabbitmq_use_ssl: false
-    galera_client_drop_config_file: false

tests/test-install-neutron.yml

@@ -95,57 +95,5 @@
     - role: "{{ rolename | basename }}"
   post_tasks:
     - include: test-neutron-functional.yml
-  vars:
+  vars_files:
-    external_lb_vip_address: 10.100.100.3
+    - test-vars.yml
-    internal_lb_vip_address: 10.100.100.3
-    neutron_galera_address: 10.100.100.2
-    neutron_galera_database: neutron
-    galera_root_password: secrete
-    galera_client_drop_config_file: false
-    neutron_rabbitmq_password: secrete
-    neutron_rabbitmq_userid: neutron
-    neutron_rabbitmq_vhost: /neutron
-    rabbitmq_servers: 10.100.100.2
-    rabbitmq_use_ssl: true
-    rabbitmq_port: 5671
-    keystone_admin_user_name: admin
-    keystone_admin_tenant_name: admin
-    keystone_auth_admin_password: SuperSecretePassword
-    keystone_service_adminuri_insecure: false
-    keystone_service_internaluri_insecure: false
-    keystone_service_internaluri: "http://{{ internal_lb_vip_address }}:5000"
-    keystone_service_internalurl: "{{ keystone_service_internaluri }}/v3"
-    keystone_service_adminuri: "http://{{ internal_lb_vip_address }}:35357"
-    keystone_service_adminurl: "{{ keystone_service_adminuri }}/v3"
-    keystone_service_region: RegionOne
-    nova_keystone_auth_plugin: password
-    nova_metadata_port: 8775
-    nova_metadata_proxy_secret: secrete
-    nova_service_adminurl: "http://{{ internal_lb_vip_address }}:8774"
-    nova_service_password: secrete
-    nova_service_project_domain_id: default
-    nova_service_project_name: service
-    nova_service_region: RegionOne
-    nova_service_user_domain_id: default
-    nova_service_user_name: nova
-    neutron_venv_tag: testing
-    neutron_git_install_branch: stable/mitaka
-    neutron_requirements_git_install_branch: stable/mitaka
-    neutron_developer_mode: true
-    neutron_provider_networks:
-      network_flat_networks: "flat"
-      network_mappings: "flat:{{ ansible_default_ipv4.interface }}"
-      network_types: "vxlan,flat,vlan"
-      network_vlan_ranges: "vlan:1:1,vlan:1024:1025"
-      network_vxlan_ranges: "1:1000"
-    neutron_ha_vrrp_auth_password: secrete
-    neutron_service_password: secrete
-    neutron_container_mysql_password: SuperSecrete
-    neutron_management_address: "{{ internal_lb_vip_address }}"
-    openrc_os_auth_url: "http://127.0.0.1:5000/v3"
-    openrc_os_password: "{{ keystone_auth_admin_password }}"
-    openrc_os_domain_name: Default
-    memcached_servers: 127.0.0.1
-    memcached_encryption_key: secrete
-    debug: true
-    verbose: true

tests/test-prepare-host.yml

@@ -15,15 +15,14 @@
 
 - name: Perform basic LXC host setup
   hosts: localhost
-  connection: local
   pre_tasks:
     - name: Ensure root's new public ssh key is in authorized_keys
       authorized_key:
         user: root
-        key: "{{ hostvars['127.0.0.1']['lxc_container_ssh_key'] }}"
+        key: "{{ hostvars['localhost']['lxc_container_ssh_key'] }}"
         manage_dir: no
     - set_fact:
-        lxc_container_ssh_key: "{{ hostvars['127.0.0.1']['lxc_container_ssh_key'] }}"
+        lxc_container_ssh_key: "{{ hostvars['localhost']['lxc_container_ssh_key'] }}"
     - name: Check if this is an OpenStack-CI nodepool instance
       stat:
         path: /etc/nodepool/provider
@@ -54,36 +53,3 @@
           name: "trusty.tgz"
           sha256sum: "56c6a6e132ea7d10be2f3e8104f47136ccf408b30e362133f0dc4a0a9adb4d0c"
           chroot_path: trusty/rootfs-amd64
-    - role: "py_from_git"
-      git_repo: "https://github.com/lxc/python2-lxc"
-      git_dest: "/opt/lxc_python2"
-      git_install_branch: "master"
-  post_tasks:
-    # THIS TASK IS ONLY BEING DONE BECAUSE THE TOX SHARED LXC LIB IS NOT USABLE ON A
-    #  HOST MACHINE THAT MAY NOT HAVE ACCESS TO THE VENV.
-    - name: Ensure the lxc lib is on the host
-      command: /usr/local/bin/pip install /opt/lxc_python2
-    # Inventory is being pre-loaded using a post tasks instead of through a dynamic
-    #  inventory system. While this is not a usual method for deployment it's being
-    #  done for functional testing.
-    - name: Create container hosts
-      add_host:
-        groups: "{{ item.groups }}"
-        hostname: "{{ item.name }}"
-        inventory_hostname: "{{ item.name }}"
-        ansible_ssh_host: "{{ item.address }}"
-        ansible_become: true
-        properties:
-          service_name: "{{ item.service }}"
-        container_networks:
-          management_address:
-            address: "{{ item.address }}"
-            bridge: "lxcbr0"
-            interface: "eth1"
-            netmask: "255.255.252.0"
-            type: "veth"
-        physical_host: localhost
-        container_name: "{{ item.name }}"
-      with_items:
-        - { name: "infra1", service: "infra1", address: "10.100.100.2", groups: "all,all_containers,rabbitmq_all,galera_all,service_all" }
-        - { name: "openstack1", service: "openstack1", address: "10.100.100.3", groups: "all,all_containers,keystone_all,neutron_agent,neutron_dhcp_agent,neutron_linuxbridge_agent,neutron_metering_agent,neutron_l3_agent,neutron_lbaas_agent,neutron_metadata_agent,neutron_server,neutron_all" }

tests/test-prepare-keys.yml

@@ -13,10 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# NOTE: we use become_user because setting become: no or become: false
+#       doesn't seem to override the ansible_become=true in the
+#       inventory
 - name: Create ssh key pairs for use with containers
-  hosts: 127.0.0.1
+  hosts: localhost
-  connection: local
+  become_user: "{{ ansible_ssh_user }}"
-  become: false
   tasks:
     - name: Create ssh key pair for root
       user:

tests/test-vars.yml

@@ -0,0 +1,65 @@
+debug: true
+external_lb_vip_address: 10.100.100.3
+galera_client_drop_config_file: false
+galera_client_drop_config_file: false
+galera_root_password: secrete
+internal_lb_vip_address: 10.100.100.3
+keystone_admin_tenant_name: admin
+keystone_admin_user_name: admin
+keystone_auth_admin_password: SuperSecretePassword
+keystone_container_mysql_password: SuperSecrete
+keystone_developer_mode: true
+keystone_galera_address: 10.100.100.2
+keystone_galera_database: keystone
+keystone_git_install_branch: stable/mitaka
+keystone_rabbitmq_password: secrete
+keystone_rabbitmq_port: 5671
+keystone_rabbitmq_servers: 10.100.100.2
+keystone_rabbitmq_use_ssl: false
+keystone_rabbitmq_userid: keystone
+keystone_rabbitmq_vhost: /keystone
+keystone_requirements_git_install_branch: stable/mitaka
+keystone_service_adminuri: "http://{{ internal_lb_vip_address }}:35357"
+keystone_service_adminuri_insecure: false
+keystone_service_adminurl: "{{ keystone_service_adminuri }}/v3"
+keystone_service_internaluri: "http://{{ internal_lb_vip_address }}:5000"
+keystone_service_internaluri_insecure: false
+keystone_service_internalurl: "{{ keystone_service_internaluri }}/v3"
+keystone_service_password: secrete
+keystone_service_region: RegionOne
+keystone_venv_tag: testing
+memcached_encryption_key: secrete
+memcached_servers: 127.0.0.1
+neutron_container_mysql_password: SuperSecrete
+neutron_developer_mode: true
+neutron_galera_address: 10.100.100.2
+neutron_galera_database: neutron
+neutron_git_install_branch: stable/mitaka
+neutron_ha_vrrp_auth_password: secrete
+neutron_management_address: "{{ internal_lb_vip_address }}"
+neutron_provider_networks:
+  network_types: "vxlan"
+  network_vxlan_ranges: "1:1000"
+neutron_rabbitmq_password: secrete
+neutron_rabbitmq_userid: neutron
+neutron_rabbitmq_vhost: /neutron
+neutron_requirements_git_install_branch: stable/mitaka
+neutron_service_password: secrete
+neutron_venv_tag: testing
+nova_keystone_auth_plugin: password
+nova_metadata_port: 8775
+nova_metadata_proxy_secret: secrete
+nova_service_adminurl: "http://{{ internal_lb_vip_address }}:8774"
+nova_service_password: secrete
+nova_service_project_domain_id: default
+nova_service_project_name: service
+nova_service_region: RegionOne
+nova_service_user_domain_id: default
+nova_service_user_name: nova
+openrc_os_auth_url: "http://127.0.0.1:5000/v3"
+openrc_os_domain_name: Default
+openrc_os_password: "{{ keystone_auth_admin_password }}"
+rabbitmq_port: 5671
+rabbitmq_servers: 10.100.100.2
+rabbitmq_use_ssl: true
+verbose: true