CI: Debug init-runonce

Also adds gawk for timestamping.

This helps to correlate init events with failures elsewhere.

Change-Id: I22fdb683ecf9870b2d66fedd6b40b7004317130a

tests/init-core-openstack.sh

@@ -2,6 +2,7 @@
 
 set -o xtrace
 set -o errexit
+set -o pipefail
 
 export PYTHONUNBUFFERED=1
 
@@ -11,7 +12,7 @@ function init_runonce {
     . ~/openstackclient-venv/bin/activate
 
     echo "Initialising OpenStack resources via init-runonce"
-    tools/init-runonce &> /tmp/logs/ansible/init-runonce
+    KOLLA_DEBUG=1 tools/init-runonce |& gawk '{ print strftime("%F %T"), $0; }' &> /tmp/logs/ansible/init-runonce
 
     echo "Setting address on the external network bridge"
     if [[ $SCENARIO == "linuxbridge" ]]; then

tests/pre.yml

@@ -23,6 +23,13 @@
         vxlan_interface_name: "{{ neutron_external_interface_name }}"
         vxlan_vni: 10001
   tasks:
+    # NOTE(yoctozepto): we use gawk to add time to each logged line
+    # outside of Ansible (e.g. for init-runonce)
+    - name: Install gawk
+      package:
+        name: gawk
+      become: true
+
     - name: Ensure /tmp/logs/ dir
       file:
         path: "{{ logs_dir }}"

tools/init-runonce

@@ -1,6 +1,16 @@
 #!/bin/bash
 
 set -o errexit
+set -o pipefail
+
+KOLLA_DEBUG=${KOLLA_DEBUG:-0}
+
+KOLLA_OPENSTACK_COMMAND=openstack
+
+if [[ $KOLLA_DEBUG -eq 1 ]]; then
+    set -o xtrace
+    KOLLA_OPENSTACK_COMMAND="$KOLLA_OPENSTACK_COMMAND --debug"
+fi
 
 # This script is meant to be run once after running start for the first
 # time.  This script downloads a cirros image and registers it.  Then it
@@ -44,7 +54,7 @@ if [[ "${OS_USERNAME}" == "" ]]; then
 fi
 
 # Test to ensure configure script is run only once
-if openstack image list | grep -q cirros; then
+if $KOLLA_OPENSTACK_COMMAND image list | grep -q cirros; then
     echo "This tool should only be run once per deployment."
     exit
 fi
@@ -63,42 +73,42 @@ else
 fi
 
 echo Creating glance image.
-openstack image create --disk-format qcow2 --container-format bare --public \
+$KOLLA_OPENSTACK_COMMAND image create --disk-format qcow2 --container-format bare --public \
     --property os_type=${IMAGE_TYPE} --file ${IMAGE_PATH}/${IMAGE} ${IMAGE_NAME}
 
 echo Configuring neutron.
 if [[ $ENABLE_EXT_NET -eq 1 ]]; then
-    openstack network create --external --provider-physical-network physnet1 \
+    $KOLLA_OPENSTACK_COMMAND network create --external --provider-physical-network physnet1 \
         --provider-network-type flat public1
-    openstack subnet create --no-dhcp \
+    $KOLLA_OPENSTACK_COMMAND subnet create --no-dhcp \
         --allocation-pool ${EXT_NET_RANGE} --network public1 \
         --subnet-range ${EXT_NET_CIDR} --gateway ${EXT_NET_GATEWAY} public1-subnet
 fi
 
-openstack network create --provider-network-type vxlan demo-net
+$KOLLA_OPENSTACK_COMMAND network create --provider-network-type vxlan demo-net
-openstack subnet create --subnet-range 10.0.0.0/24 --network demo-net \
+$KOLLA_OPENSTACK_COMMAND subnet create --subnet-range 10.0.0.0/24 --network demo-net \
     --gateway 10.0.0.1 --dns-nameserver 8.8.8.8 demo-subnet
 
-openstack router create demo-router
+$KOLLA_OPENSTACK_COMMAND router create demo-router
-openstack router add subnet demo-router demo-subnet
+$KOLLA_OPENSTACK_COMMAND router add subnet demo-router demo-subnet
 if [[ $ENABLE_EXT_NET -eq 1 ]]; then
-  openstack router set --external-gateway public1 demo-router
+  $KOLLA_OPENSTACK_COMMAND router set --external-gateway public1 demo-router
 fi
 
 # Get admin user and tenant IDs
-ADMIN_USER_ID=$(openstack user list | awk '/ admin / {print $2}')
+ADMIN_USER_ID=$($KOLLA_OPENSTACK_COMMAND user list | awk '/ admin / {print $2}')
-ADMIN_PROJECT_ID=$(openstack project list | awk '/ admin / {print $2}')
+ADMIN_PROJECT_ID=$($KOLLA_OPENSTACK_COMMAND project list | awk '/ admin / {print $2}')
-ADMIN_SEC_GROUP=$(openstack security group list --project ${ADMIN_PROJECT_ID} | awk '/ default / {print $2}')
+ADMIN_SEC_GROUP=$($KOLLA_OPENSTACK_COMMAND security group list --project ${ADMIN_PROJECT_ID} | awk '/ default / {print $2}')
 
 # Sec Group Config
-openstack security group rule create --ingress --ethertype IPv4 \
+$KOLLA_OPENSTACK_COMMAND security group rule create --ingress --ethertype IPv4 \
     --protocol icmp ${ADMIN_SEC_GROUP}
-openstack security group rule create --ingress --ethertype IPv4 \
+$KOLLA_OPENSTACK_COMMAND security group rule create --ingress --ethertype IPv4 \
     --protocol tcp --dst-port 22 ${ADMIN_SEC_GROUP}
 # Open heat-cfn so it can run on a different host
-openstack security group rule create --ingress --ethertype IPv4 \
+$KOLLA_OPENSTACK_COMMAND security group rule create --ingress --ethertype IPv4 \
     --protocol tcp --dst-port 8000 ${ADMIN_SEC_GROUP}
-openstack security group rule create --ingress --ethertype IPv4 \
+$KOLLA_OPENSTACK_COMMAND security group rule create --ingress --ethertype IPv4 \
     --protocol tcp --dst-port 8080 ${ADMIN_SEC_GROUP}
 
 if [ ! -f ~/.ssh/id_rsa.pub ]; then
@@ -107,27 +117,27 @@ if [ ! -f ~/.ssh/id_rsa.pub ]; then
 fi
 if [ -r ~/.ssh/id_rsa.pub ]; then
     echo Configuring nova public key and quotas.
-    openstack keypair create --public-key ~/.ssh/id_rsa.pub mykey
+    $KOLLA_OPENSTACK_COMMAND keypair create --public-key ~/.ssh/id_rsa.pub mykey
 fi
 
 # Increase the quota to allow 40 m1.small instances to be created
 
 # 40 instances
-openstack quota set --instances 40 ${ADMIN_PROJECT_ID}
+$KOLLA_OPENSTACK_COMMAND quota set --instances 40 ${ADMIN_PROJECT_ID}
 
 # 40 cores
-openstack quota set --cores 40 ${ADMIN_PROJECT_ID}
+$KOLLA_OPENSTACK_COMMAND quota set --cores 40 ${ADMIN_PROJECT_ID}
 
 # 96GB ram
-openstack quota set --ram 96000 ${ADMIN_PROJECT_ID}
+$KOLLA_OPENSTACK_COMMAND quota set --ram 96000 ${ADMIN_PROJECT_ID}
 
 # add default flavors, if they don't already exist
-if ! openstack flavor list | grep -q m1.tiny; then
+if ! $KOLLA_OPENSTACK_COMMAND flavor list | grep -q m1.tiny; then
-    openstack flavor create --id 1 --ram 512 --disk 1 --vcpus 1 m1.tiny
+    $KOLLA_OPENSTACK_COMMAND flavor create --id 1 --ram 512 --disk 1 --vcpus 1 m1.tiny
-    openstack flavor create --id 2 --ram 2048 --disk 20 --vcpus 1 m1.small
+    $KOLLA_OPENSTACK_COMMAND flavor create --id 2 --ram 2048 --disk 20 --vcpus 1 m1.small
-    openstack flavor create --id 3 --ram 4096 --disk 40 --vcpus 2 m1.medium
+    $KOLLA_OPENSTACK_COMMAND flavor create --id 3 --ram 4096 --disk 40 --vcpus 2 m1.medium
-    openstack flavor create --id 4 --ram 8192 --disk 80 --vcpus 4 m1.large
+    $KOLLA_OPENSTACK_COMMAND flavor create --id 4 --ram 8192 --disk 80 --vcpus 4 m1.large
-    openstack flavor create --id 5 --ram 16384 --disk 160 --vcpus 8 m1.xlarge
+    $KOLLA_OPENSTACK_COMMAND flavor create --id 5 --ram 16384 --disk 160 --vcpus 8 m1.xlarge
 fi
 
 cat << EOF