Add a job for testing deployment of bifrost

Deploys a bifrost container using kolla-ansible bifrost-deploy.

IPA and disk images are downloaded rather than built to improve
reliability.

Currently only minimal testing of the deployment is performed, creating
and deleting an ironic node. Ideally we would perform a bare metal node
deployment.

The job is based on CentOS, as Ubuntu bifrost deployment is currently
failing with a python-MySQLdb error.

Change-Id: Ic45094594c21116b5b0d6a606f568fc7954175e3

.zuul.yaml

@@ -8,6 +8,7 @@
         - kolla-ansible-ubuntu-source-ceph
         - kolla-ansible-centos-source-ceph
         - kolla-ansible-oraclelinux-source-ceph
+        - kolla-ansible-bifrost-centos-source
 
         - openstack-tox-lower-constraints
     gate:
@@ -81,6 +82,13 @@
     roles:
       - zuul: openstack-infra/zuul-jobs
 
+- job:
+    name: kolla-ansible-bifrost-base
+    parent: kolla-ansible-base
+    vars:
+      scenario: bifrost
+      install_type: source
+
 - job:
     name: kolla-ansible-centos-source
     parent: kolla-ansible-base
@@ -164,3 +172,11 @@
       base_distro: oraclelinux
       install_type: source
       scenario: ceph
+
+- job:
+    name: kolla-ansible-bifrost-centos-source
+    parent: kolla-ansible-bifrost-base
+    nodeset: kolla-ansible-centos
+    voting: false
+    vars:
+      base_distro: centos

tests/get_logs.sh

@@ -46,6 +46,16 @@ copy_logs() {
         docker exec ceph_mon ceph osd tree > ${LOG_DIR}/kolla/ceph/ceph_osd_tree.txt
     fi
 
+    # bifrost related logs
+    if [[ $(docker ps --filter name=bifrost_deploy --format "{{.Names}}") ]]; then
+        for service in dnsmasq ironic-api ironic-conductor ironic-inspector mariadb nginx rabbitmq-server; do
+            mkdir -p ${LOG_DIR}/kolla/$service
+            docker exec bifrost_deploy systemctl status $service > ${LOG_DIR}/kolla/$service/systemd-status-$service.txt
+        done
+        docker exec bifrost_deploy journalctl -u mariadb > ${LOG_DIR}/kolla/mariadb/mariadb.txt
+        docker exec bifrost_deploy journalctl -u rabbitmq-server > ${LOG_DIR}/kolla/rabbitmq-server/rabbitmq.txt
+    fi
+
     for container in $(docker ps -a --format "{{.Names}}"); do
         docker logs --tail all ${container} > ${LOG_DIR}/docker_logs/${container}.txt
     done

tests/run.yml

@@ -62,6 +62,7 @@
         path: "/etc/kolla/config/nova"
         state: "directory"
         mode: 0777
+      when: scenario != "bifrost"
       become: true
       delegate_to: "primary"
 
@@ -69,6 +70,23 @@
       template:
         src: "{{ kolla_ansible_full_src_dir }}/tests/templates/nova-compute-overrides.j2"
         dest: /etc/kolla/config/nova/nova-compute.conf
+      when: scenario != "bifrost"
+      delegate_to: "primary"
+
+    - name: ensure bifrost conf overrides dir exists
+      file:
+        path: "/etc/kolla/config/bifrost"
+        state: "directory"
+        mode: 0777
+      when: scenario == "bifrost"
+      become: true
+      delegate_to: "primary"
+
+    - name: generate bifrost DIB config overrides
+      template:
+        src: "{{ kolla_ansible_full_src_dir }}/tests/templates/bifrost-dib-overrides.j2"
+        dest: /etc/kolla/config/bifrost/dib.yml
+      when: scenario == "bifrost"
       delegate_to: "primary"
 
     - name: copy passwords.yml file

tests/templates/bifrost-dib-overrides.j2

@@ -0,0 +1,7 @@
+---
+# Don't build an IPA deployment image, instead download upstream images.
+create_ipa_image: false
+download_ipa: true
+
+# Don't build a disk image. It takes time and can be unreliable.
+use_cirros: true

tests/templates/globals-default.j2

@@ -6,12 +6,14 @@ kolla_install_type: "{{ install_type }}"
 # in the CI gate.
 keepalived_virtual_router_id: "{{ 250 | random(1) }}"
 
+{% if scenario != "bifrost" %}
 {% if hostvars|length > 2 %}
 kolla_internal_vip_address: "{{ api_interface_address }}"
 enable_haproxy: "no"
 {% else %}
 kolla_internal_vip_address: "169.254.169.10"
 {% endif %}
+{% endif %}
 
 network_interface: "{{ api_interface_name }}"
 docker_restart_policy: "never"
@@ -24,11 +26,13 @@ docker_registry: "{{ api_interface_address }}:4000"
 docker_namespace: "kolla"
 openstack_release: "{{ zuul.branch | basename }}"
 {% endif %}
+{% if scenario != "bifrost" %}
 neutron_external_interface: "fake_interface"
 enable_horizon: "yes"
 enable_heat: "no"
 openstack_logging_debug: "True"
 openstack_service_workers: "1"
+{% endif %}
 
 # This is experimental feature, disable if gate fail.
 glance_enable_rolling_upgrade: "yes"

tools/setup_gate.sh

@@ -34,10 +34,13 @@ EOF
     echo "RUN echo $(base64 -w0 ${PIP_CONF}) | base64 -d > /etc/pip.conf" | sudo tee /etc/kolla/header
     rm ${PIP_CONF}
 
+    if [[ $ACTION != "bifrost" ]]; then
         GATE_IMAGES="cron,fluentd,glance,haproxy,keepalived,keystone,kolla-toolbox,mariadb,memcached,neutron,nova,openvswitch,rabbitmq,horizon"
+    else
+        GATE_IMAGES="bifrost"
+    fi
 
-# TODO(jeffrey4l): this doesn't work with zuulv3
-if echo $ACTION | grep -q "ceph"; then
+    if [[ $ACTION == "ceph" ]]; then
         GATE_IMAGES+=",ceph,cinder"
     fi
 
@@ -134,6 +137,70 @@ function sanity_check {
     fi
 }
 
+function test_openstack {
+    # Create dummy interface for neutron
+    ansible -m shell -i ${RAW_INVENTORY} -a "ip l a fake_interface type dummy" all
+
+    #TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
+    sudo chmod -R 777 /etc/kolla
+    # Actually do the deployment
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks1
+    # TODO(jeffrey4l): add pull action when we have a local registry
+    # service in CI
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy
+
+    # Test OpenStack Environment
+    # TODO: use kolla-ansible check when it's ready
+
+    sanity_check
+
+    # TODO(jeffrey4l): make some configure file change and
+    # trigger a real reconfigure
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure >  /tmp/logs/ansible/post-deploy
+    # TODO(jeffrey4l): need run a real upgrade
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade
+
+    # run prechecks again
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks2
+}
+
+function sanity_check_bifrost {
+    # TODO(mgoddard): More testing, deploy bare metal nodes.
+    # TODO(mgoddard): Use openstackclient when clouds.yaml works. See
+    # https://bugs.launchpad.net/bifrost/+bug/1754070.
+    attempts=0
+    while [[ $(sudo docker exec bifrost_deploy bash -c "source env-vars && ironic driver-list" | wc -l) -le 4 ]]; do
+        attempts=$((attempts + 1))
+        if [[ $attempts -gt 6 ]]; then
+            echo "Timed out waiting for ironic conductor to become active"
+            exit 1
+        fi
+        sleep 10
+    done
+    sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-list"
+    sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-create --driver ipmi --name test-node"
+    sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-delete test-node"
+}
+
+function test_bifrost {
+    # TODO(mgoddard): run prechecks.
+
+    # Deploy the bifrost container.
+    # TODO(mgoddard): add pull action when we have a local registry service in
+    # CI.
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost
+
+    # Test Bifrost Environment
+    sanity_check_bifrost
+
+    # TODO(mgoddard): make some configuration file changes and trigger a real
+    # reconfigure.
+    tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost >  /tmp/logs/ansible/deploy-bifrost2
+
+    # TODO(mgoddard): perform an upgrade.
+}
+
 check_failure() {
     # All docker container's status are created, restarting, running, removing,
     # paused, exited and dead. Containers without running status are treated as
@@ -160,30 +227,10 @@ setup_node
 tools/kolla-ansible -i ${RAW_INVENTORY} bootstrap-servers > /tmp/logs/ansible/bootstrap-servers
 prepare_images
 
-# Create dummy interface for neutron
-ansible -m shell -i ${RAW_INVENTORY} -a "ip l a fake_interface type dummy" all
-
-#TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
-sudo chmod -R 777 /etc/kolla
-# Actually do the deployment
-tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks1
-# TODO(jeffrey4l): add pull action when we have a local registry
-# service in CI
-tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy
-tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy
-
-# Test OpenStack Environment
-# TODO: use kolla-ansible check when it's ready
-
-sanity_check
-
-# TODO(jeffrey4l): make some configure file change and
-# trigger a real reconfigure
-tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure >  /tmp/logs/ansible/post-deploy
-# TODO(jeffrey4l): need run a real upgrade
-tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade
-
-# run prechecks again
-tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks2
+if [[ $ACTION != bifrost ]]; then
+    test_openstack
+else
+    test_bifrost
+fi
 
 check_failure