From 2c1c54d7b7b03466df07bc2977e8dab948c56796 Mon Sep 17 00:00:00 2001 From: Julia Kreger Date: Tue, 12 May 2015 17:34:06 -0400 Subject: [PATCH] Fix playbooks for OpenStack CI The OpenStack CI systems are preconfigured with various components such as databases, message queues, caching dns resolvers. The CI systems also lack things like public/private key pairs, and known_hosts files. Due to this, a number of minor changes are required in Bifrost's test sequence to account for and route around these differences in order to allow test jobs to run until completion. This also demonstrated that some steps have been operating under the assumption that the user likely already had certain things installed, so service restart tweeks have also been necessary. Added a step to restart libvirt-bin in order to allow it to pickup new capabilities due to the install of the qemu emulator. Added additional steps to collect kernel log and `virsh capabilities` output. Removed MySQL step nolog statements as the password is automatically suppressed and it suppresses information that could be required for troubleshooting. Added in specific username/password logic for MySQL steps if ZUUL is detected in order to use the OpenStack CI defaults. Changed VM test node creation to occur upon every attempt. Extended the startup wait for a test node to be 900 seconds from 600 seconds. Added collection of process list and listening sockets if ZUUL is detected. Added report of VM console log and various commands to assist in troubleshooting a failing job. Added auto-generation of a user ssh keypair if absent when operating in OpenStack CI. Added explicit adjustment of iptables to permit connections on the network interface that the hosts are being provisioned via on 8080/tcp and 6385/tcp. Change-Id: I3c37c5c21af0aefb5007c5775043f7f837389c2d --- .../tasks/ssh_public_key_path.yaml | 10 ++++++++ .../bifrost-create-vm-nodes/tasks/main.yml | 16 +++++++++++- .../bifrost-prepare-for-test/tasks/main.yml | 2 +- playbooks/roles/ironic-enroll/tasks/main.yml | 3 +++ playbooks/roles/ironic-install/tasks/main.yml | 19 +++++++++++--- .../ironic-install/templates/dnsmasq.conf.j2 | 4 +++ playbooks/test-bifrost.yaml | 10 ++++++++ scripts/test-bifrost.sh | 25 +++++++++++++++++++ 8 files changed, 83 insertions(+), 6 deletions(-) diff --git a/playbooks/roles/bifrost-configdrives/tasks/ssh_public_key_path.yaml b/playbooks/roles/bifrost-configdrives/tasks/ssh_public_key_path.yaml index ccfaff6e2..1147d21c9 100644 --- a/playbooks/roles/bifrost-configdrives/tasks/ssh_public_key_path.yaml +++ b/playbooks/roles/bifrost-configdrives/tasks/ssh_public_key_path.yaml @@ -12,6 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. --- +- name: "Determine if OpenStack CI is missing an SSH key" + stat: path={{ssh_public_key_path}} + register: test_ssh_public_key_path + when: ci_testing_zuul is defined +- name: "Create an SSH key for Jenkins user if operating in OpenStack CI" + shell: ssh-keygen -f ~/.ssh/id_rsa -N "" + when: ci_testing_zuul is defined and test_ssh_public_key_path.stat.exists == false +- name: "Create an empty ssh known_hosts file for Jenkins user if operating in OpenStack CI" + shell: touch ~/.ssh/known_hosts && chmod 600 ~/.ssh/known_hosts + when: ci_testing_zuul is defined and test_ssh_public_key_path.stat.exists == false - name: "Defined ssh_public_key_path - Check to see if there is a file where the ssh_public_key_path is defined" local_action: stat path={{ ssh_public_key_path }} register: test_ssh_public_key_path diff --git a/playbooks/roles/bifrost-create-vm-nodes/tasks/main.yml b/playbooks/roles/bifrost-create-vm-nodes/tasks/main.yml index b97106772..54d2a2ee7 100644 --- a/playbooks/roles/bifrost-create-vm-nodes/tasks/main.yml +++ b/playbooks/roles/bifrost-create-vm-nodes/tasks/main.yml @@ -20,12 +20,26 @@ - libvirt-bin - qemu-utils - qemu-kvm + - qemu-system-x86 - sgabios +- name: "Restart libvirt service" + service: name=libvirt-bin state=restarted - name: "Create virtual machines" - script: create_vm_nodes-for-role.sh creates="{{baremetal_csv_file}}" + script: create_vm_nodes-for-role.sh environment: NODEOUTPUT: "{{baremetal_csv_file}}" + register: task_create_vm_nodes + ignore_errors: yes delegate_to: localhost +- name: "Execute `dmesg` to collect debugging output should VM creation fail." + command: dmesg + when: task_create_vm_nodes.rc != 0 +- name: "Execute `virsh capabilities` to collect debugging output should VM creation fail." + command: virsh capabilities + when: task_create_vm_nodes.rc != 0 +- name: "Abort due to failed VM creation" + fail: msg="VM Creation step failed, please review dmesg output for additional details" + when: task_create_vm_nodes.rc != 0 - name: "Setting file permissions such that the baremetal csv file at /tmp/baremetal.csv can be read by the user executing Ansible" file: path="{{baremetal_csv_file}}" owner="{{ansible_env.SUDO_USER}}" when: ansible_env.SUDO_USER is defined and baremetal_csv_file is defined and baremetal_csv_file != "" diff --git a/playbooks/roles/bifrost-prepare-for-test/tasks/main.yml b/playbooks/roles/bifrost-prepare-for-test/tasks/main.yml index 1a892a7e1..26fcf51da 100644 --- a/playbooks/roles/bifrost-prepare-for-test/tasks/main.yml +++ b/playbooks/roles/bifrost-prepare-for-test/tasks/main.yml @@ -13,7 +13,7 @@ # limitations under the License. --- - name: "Waiting for the base testvm machine to become available." - wait_for: state=started port=22 host={{ item.split(',')[11] }} timeout=600 + wait_for: state=started port=22 host={{ item.split(',')[11] }} timeout=900 with_lines: - cat {{ baremetal_csv_file }} - name: "Adding testvms to Ansible Inventory" diff --git a/playbooks/roles/ironic-enroll/tasks/main.yml b/playbooks/roles/ironic-enroll/tasks/main.yml index 1d2f09443..d3dc31b17 100644 --- a/playbooks/roles/ironic-enroll/tasks/main.yml +++ b/playbooks/roles/ironic-enroll/tasks/main.yml @@ -21,6 +21,9 @@ - name: "Error if file does not exist." fail: msg="The variable defined for baremetal_csv_file is not to a file. Please define a file and try again." when: test_baremetal_csv_file.stat.isreg == false +- name: "Report the contents of the baremetal_csv_file if we are running in OpenStack CI" + command: cat "{{ baremetal_csv_file }}" + when: ci_testing_zuul is defined - name: "If testing, enroll virtual machines." include: virtual_enroll.yaml when: testing == true diff --git a/playbooks/roles/ironic-install/tasks/main.yml b/playbooks/roles/ironic-install/tasks/main.yml index e4f54e13c..53c78b0ef 100644 --- a/playbooks/roles/ironic-install/tasks/main.yml +++ b/playbooks/roles/ironic-install/tasks/main.yml @@ -82,10 +82,17 @@ - name: "MySQL - Creating DB" mysql_db: login_user=root login_password={{ mysql_password }} name=ironic state=present encoding=utf8 register: test_created_db - no_log: True + when: ci_testing_zuul is not defined - name: "MySQL - Creating user for Ironic" mysql_user: login_user=root login_password={{ mysql_password }} name=ironic password={{ ironic_db_password }} priv=ironic.*:ALL state=present - no_log: True + when: ci_testing_zuul is not defined +- name: "MySQL - Creating DB - OpenStack CI" + mysql_db: login_user=openstack_citest login_password=openstack_citest name=ironic state=present encoding=utf8 + register: test_created_db + when: ci_testing_zuul is defined +- name: "MySQL - Creating user for Ironic - OpenStack CI" + mysql_user: login_user=openstack_citest login_password=openstack_citest name=ironic password={{ ironic_db_password }} priv=ironic.*:ALL state=present + when: ci_testing_zuul is defined - name: "Install Ironic using pip" pip: name=/opt/stack/ironic state=latest when: skip_install is not defined @@ -195,8 +202,8 @@ when: test_sgabios_qemu == false and test_sgabios_misc == true and testing == true - name: "Deploying nginx configuration file for serving HTTP requests" template: src=nginx.conf.j2 dest=/etc/nginx/nginx.conf -- name: "Ensuring dnsmasq is running" - service: name={{ item }} state=started +- name: "Ensuring services are running with current config" + service: name={{ item }} state=restarted with_items: - xinetd - dnsmasq @@ -250,3 +257,7 @@ - name: "Creating image via disk imae builder" include: create_dib_image.yml when: test_os_image_present.stat.exists == false and transform_boot_image == false and create_image_via_dib == true +- name: "Explicitly permit TCP/8080 for file downloads from nodes to be provisioned" + command: iptables -I INPUT -p tcp --dport 8080 -i {{network_interface}} -j ACCEPT +- name: "Explicitly permit TCP/6385 for IPA callback" + command: iptables -I INPUT -p tcp --dport 6385 -i {{network_interface}} -j ACCEPT diff --git a/playbooks/roles/ironic-install/templates/dnsmasq.conf.j2 b/playbooks/roles/ironic-install/templates/dnsmasq.conf.j2 index 88353dd93..c3328f564 100644 --- a/playbooks/roles/ironic-install/templates/dnsmasq.conf.j2 +++ b/playbooks/roles/ironic-install/templates/dnsmasq.conf.j2 @@ -7,7 +7,11 @@ # Listen on this specific port instead of the standard DNS port # (53). Setting this to zero completely disables DNS function, # leaving only DHCP and/or TFTP. +{% if ci_testing_zuul is defined %} +port=0 +{% else %} port=53 +{% endif %} # The following two options make you a better netizen, since they # tell dnsmasq to filter out queries which the public DNS cannot diff --git a/playbooks/test-bifrost.yaml b/playbooks/test-bifrost.yaml index dae61753e..47c4561c1 100644 --- a/playbooks/test-bifrost.yaml +++ b/playbooks/test-bifrost.yaml @@ -13,6 +13,16 @@ set_fact: ci_testing: true when: lookup('env', 'ZUUL_CHANGES') != "" + - name: "Set ci_testing_zuul if it appears we are running in upstream OpenStack CI" + set_fact: + ci_testing_zuul: true + when: "'bare-trusty' in ansible_hostname" + - name: "Collect process list if running in OpenStack CI" + command: ps aux + when: ci_testing_zuul is defined + - name: "Collect list of listening network sockets if running in OpenStack CI" + shell: netstat -apn|grep LISTEN + when: ci_testing_zuul is defined roles: - role: bifrost-create-vm-nodes - role: ironic-install diff --git a/scripts/test-bifrost.sh b/scripts/test-bifrost.sh index fe687f992..89f3ccf1e 100755 --- a/scripts/test-bifrost.sh +++ b/scripts/test-bifrost.sh @@ -1,6 +1,7 @@ #!/bin/bash set -eux set -o pipefail +export PYTHONUNBUFFERED=1 SCRIPT_HOME=$(dirname $0) BIFROST_HOME=$SCRIPT_HOME/.. @@ -16,5 +17,29 @@ cd $BIFROST_HOME/playbooks # Perform a syntax check ansible-playbook -vvvv -i inventory/localhost test-bifrost.yaml --syntax-check --list-tasks +set +e + # Execute test playbook ansible-playbook -vvvv -i inventory/localhost test-bifrost.yaml +EXITCODE=$? +if [ $EXITCODE != 0 ]; then + echo "*************************" + echo "Test failed. Test VM log:" + sudo cat /var/log/libvirt/baremetal_logs/testvm1_console.log + echo "*************************" + echo "Kernel log:" + sudo dmesg + echo "*************************" + echo "Network Sockets in LISTEN state:" + sudo netstat -apn|grep LISTEN + echo "*************************" + echo "Firewalling settings:" + sudo iptables -L -n -v + echo "*************************" + echo "Ironic API log, last 1000 lines:" + sudo cat /var/log/upstart/ironic-api.log + echo "*************************" + echo "Ironic Conductor log, last 1000 lines:" + sudo cat /var/log/upstart/ironic-conductor.log +fi +exit $EXITCODE