Fix playbooks for OpenStack CI

The OpenStack CI systems are preconfigured with various components
such as databases, message queues, caching dns resolvers.  The
CI systems also lack things like public/private key pairs, and
known_hosts files.

Due to this, a number of minor changes are required in Bifrost's test
sequence to account for and route around these differences in order
to allow test jobs to run until completion. This also demonstrated
that some steps have been operating under the assumption that the user
likely already had certain things installed, so service restart
tweeks have also been necessary.

Added a step to restart libvirt-bin in order to allow it to pickup
new capabilities due to the install of the qemu emulator.

Added additional steps to collect kernel log and `virsh capabilities`
output.

Removed MySQL step nolog statements as the password is automatically
suppressed and it suppresses information that could be required for
troubleshooting.

Added in specific username/password logic for MySQL steps if ZUUL
is detected in order to use the OpenStack CI defaults.

Changed VM test node creation to occur upon every attempt.

Extended the startup wait for a test node to be 900 seconds from 600
seconds.

Added collection of process list and listening sockets if ZUUL is
detected.

Added report of VM console log and various commands to assist in
troubleshooting a failing job.

Added auto-generation of a user ssh keypair if absent when operating
in OpenStack CI.

Added explicit adjustment of iptables to permit connections on the
network interface that the hosts are being provisioned via on
8080/tcp and 6385/tcp.

Change-Id: I3c37c5c21af0aefb5007c5775043f7f837389c2d

playbooks/roles/bifrost-configdrives/tasks/ssh_public_key_path.yaml

@@ -12,6 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ---
+- name: "Determine if OpenStack CI is missing an SSH key"
+  stat: path={{ssh_public_key_path}}
+  register: test_ssh_public_key_path
+  when: ci_testing_zuul is defined
+- name: "Create an SSH key for Jenkins user if operating in OpenStack CI"
+  shell: ssh-keygen -f ~/.ssh/id_rsa -N ""
+  when: ci_testing_zuul is defined and test_ssh_public_key_path.stat.exists == false
+- name: "Create an empty ssh known_hosts file for Jenkins user if operating in OpenStack CI"
+  shell: touch ~/.ssh/known_hosts && chmod 600 ~/.ssh/known_hosts
+  when: ci_testing_zuul is defined and test_ssh_public_key_path.stat.exists == false
 - name: "Defined ssh_public_key_path - Check to see if there is a file where the ssh_public_key_path is defined"
   local_action: stat path={{ ssh_public_key_path }}
   register: test_ssh_public_key_path

playbooks/roles/bifrost-create-vm-nodes/tasks/main.yml

@@ -20,12 +20,26 @@
     - libvirt-bin
     - qemu-utils
     - qemu-kvm
+    - qemu-system-x86
     - sgabios
+- name: "Restart libvirt service"
+  service: name=libvirt-bin state=restarted
 - name: "Create virtual machines"
-  script: create_vm_nodes-for-role.sh creates="{{baremetal_csv_file}}"
+  script: create_vm_nodes-for-role.sh
   environment:
     NODEOUTPUT: "{{baremetal_csv_file}}"
+  register: task_create_vm_nodes
+  ignore_errors: yes
   delegate_to: localhost
+- name: "Execute `dmesg` to collect debugging output should VM creation fail."
+  command: dmesg
+  when: task_create_vm_nodes.rc != 0
+- name: "Execute `virsh capabilities` to collect debugging output should VM creation fail."
+  command: virsh capabilities
+  when: task_create_vm_nodes.rc != 0
+- name: "Abort due to failed VM creation"
+  fail: msg="VM Creation step failed, please review dmesg output for additional details"
+  when: task_create_vm_nodes.rc != 0
 - name: "Setting file permissions such that the baremetal csv file at /tmp/baremetal.csv can be read by the user executing Ansible"
   file: path="{{baremetal_csv_file}}" owner="{{ansible_env.SUDO_USER}}"
   when: ansible_env.SUDO_USER is defined and baremetal_csv_file is defined and baremetal_csv_file != ""

playbooks/roles/bifrost-prepare-for-test/tasks/main.yml

@@ -13,7 +13,7 @@
 # limitations under the License.
 ---
 - name: "Waiting for the base testvm machine to become available."
-  wait_for: state=started port=22 host={{ item.split(',')[11] }} timeout=600
+  wait_for: state=started port=22 host={{ item.split(',')[11] }} timeout=900
   with_lines:
     - cat {{ baremetal_csv_file }}
 - name: "Adding testvms to Ansible Inventory"

playbooks/roles/ironic-enroll/tasks/main.yml

@@ -21,6 +21,9 @@
 - name: "Error if file does not exist."
   fail: msg="The variable defined for baremetal_csv_file is not to a file.  Please define a file and try again."
   when: test_baremetal_csv_file.stat.isreg == false
+- name: "Report the contents of the baremetal_csv_file if we are running in OpenStack CI"
+  command: cat "{{ baremetal_csv_file }}"
+  when: ci_testing_zuul is defined
 - name: "If testing, enroll virtual machines."
   include: virtual_enroll.yaml
   when: testing == true

playbooks/roles/ironic-install/tasks/main.yml

@@ -82,10 +82,17 @@
 - name: "MySQL - Creating DB"
   mysql_db: login_user=root login_password={{ mysql_password }} name=ironic state=present encoding=utf8
   register: test_created_db
-  no_log: True
+  when: ci_testing_zuul is not defined
 - name: "MySQL - Creating user for Ironic"
   mysql_user: login_user=root login_password={{ mysql_password }} name=ironic password={{ ironic_db_password }} priv=ironic.*:ALL state=present
-  no_log: True
+  when: ci_testing_zuul is not defined
+- name: "MySQL - Creating DB - OpenStack CI"
+  mysql_db: login_user=openstack_citest login_password=openstack_citest name=ironic state=present encoding=utf8
+  register: test_created_db
+  when: ci_testing_zuul is defined
+- name: "MySQL - Creating user for Ironic - OpenStack CI"
+  mysql_user: login_user=openstack_citest login_password=openstack_citest name=ironic password={{ ironic_db_password }} priv=ironic.*:ALL state=present
+  when: ci_testing_zuul is defined
 - name: "Install Ironic using pip"
   pip: name=/opt/stack/ironic state=latest
   when: skip_install is not defined
@@ -195,8 +202,8 @@
   when: test_sgabios_qemu == false and test_sgabios_misc == true and testing == true
 - name: "Deploying nginx configuration file for serving HTTP requests"
   template: src=nginx.conf.j2 dest=/etc/nginx/nginx.conf
-- name: "Ensuring dnsmasq is running"
-  service: name={{ item }} state=started
+- name: "Ensuring services are running with current config"
+  service: name={{ item }} state=restarted
   with_items:
     - xinetd
     - dnsmasq
@@ -250,3 +257,7 @@
 - name: "Creating image via disk imae builder"
   include: create_dib_image.yml
   when: test_os_image_present.stat.exists == false and transform_boot_image == false and create_image_via_dib == true
+- name: "Explicitly permit TCP/8080 for file downloads from nodes to be provisioned"
+  command: iptables -I INPUT -p tcp --dport 8080 -i {{network_interface}} -j ACCEPT
+- name: "Explicitly permit TCP/6385 for IPA callback"
+  command: iptables -I INPUT -p tcp --dport 6385 -i {{network_interface}} -j ACCEPT

playbooks/roles/ironic-install/templates/dnsmasq.conf.j2

@@ -7,7 +7,11 @@
 # Listen on this specific port instead of the standard DNS port
 # (53). Setting this to zero completely disables DNS function,
 # leaving only DHCP and/or TFTP.
+{% if ci_testing_zuul is defined %}
+port=0
+{% else %}
 port=53
+{% endif %}
 
 # The following two options make you a better netizen, since they
 # tell dnsmasq to filter out queries which the public DNS cannot

playbooks/test-bifrost.yaml

@@ -13,6 +13,16 @@
       set_fact:
          ci_testing: true
       when: lookup('env', 'ZUUL_CHANGES') != ""
+    - name: "Set ci_testing_zuul if it appears we are running in upstream OpenStack CI"
+      set_fact:
+         ci_testing_zuul: true
+      when: "'bare-trusty' in ansible_hostname"
+    - name: "Collect process list if running in OpenStack CI"
+      command: ps aux
+      when: ci_testing_zuul is defined
+    - name: "Collect list of listening network sockets if running in OpenStack CI"
+      shell: netstat -apn|grep LISTEN
+      when: ci_testing_zuul is defined
   roles:
     - role: bifrost-create-vm-nodes
     - role: ironic-install

scripts/test-bifrost.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 set -eux
 set -o pipefail
+export PYTHONUNBUFFERED=1
 
 SCRIPT_HOME=$(dirname $0)
 BIFROST_HOME=$SCRIPT_HOME/..
@@ -16,5 +17,29 @@ cd $BIFROST_HOME/playbooks
 # Perform a syntax check
 ansible-playbook -vvvv -i inventory/localhost test-bifrost.yaml --syntax-check --list-tasks
 
+set +e
+
 # Execute test playbook
 ansible-playbook -vvvv -i inventory/localhost test-bifrost.yaml
+EXITCODE=$?
+if [ $EXITCODE != 0 ]; then
+    echo "*************************"
+    echo "Test failed. Test VM log:"
+    sudo cat /var/log/libvirt/baremetal_logs/testvm1_console.log
+    echo "*************************"
+    echo "Kernel log:"
+    sudo dmesg
+    echo "*************************"
+    echo "Network Sockets in LISTEN state:"
+    sudo netstat -apn|grep LISTEN
+    echo "*************************"
+    echo "Firewalling settings:"
+    sudo iptables -L -n -v
+    echo "*************************"
+    echo "Ironic API log, last 1000 lines:"
+    sudo cat /var/log/upstart/ironic-api.log
+    echo "*************************"
+    echo "Ironic Conductor log, last 1000 lines:"
+    sudo cat /var/log/upstart/ironic-conductor.log
+fi
+exit $EXITCODE