Add CI job for ironic

Adds four new CI jobs for testing centos/ubuntu binary/source deploys
with ironic enabled. These are run only when there are changes to the
ironic role.

Performs some simple testing by creating a node using the fake-hardware
hardware type and creating a server.

Change-Id: Ie669e57ce2af53257b4ca05f45193cb73f48827a
Depends-On: https://review.opendev.org/664011

tests/get_logs.sh

@@ -7,6 +7,8 @@ copy_logs() {
 
     cp -rnL /var/lib/docker/volumes/kolla_logs/_data/* ${LOG_DIR}/kolla/
     cp -rnL /etc/kolla/* ${LOG_DIR}/kolla_configs/
+    # Don't save the IPA images.
+    rm ${LOG_DIR}/kolla_configs/config/ironic/ironic-agent.{kernel,initramfs}
     cp -rvnL /var/log/* ${LOG_DIR}/system_logs/
 
 
@@ -62,6 +64,12 @@ copy_logs() {
         docker exec haproxy bash -c 'echo show stat | socat stdio /var/lib/kolla/haproxy/haproxy.sock' > ${LOG_DIR}/kolla/haproxy/stats.txt
     fi
 
+    # FIXME: remove
+    if [[ $(docker ps -a --filter name=ironic_inspector --format "{{.Names}}") ]]; then
+        mkdir -p ${LOG_DIR}/kolla/ironic-inspector
+        ls -lR /var/lib/docker/volumes/ironic_inspector_dhcp_hosts > ${LOG_DIR}/kolla/ironic-inspector/var-lib-ls.txt
+    fi
+
     for container in $(docker ps -a --format "{{.Names}}"); do
         docker logs --tail all ${container} &> ${LOG_DIR}/docker_logs/${container}.txt
     done

tests/run.yml

@@ -98,8 +98,29 @@
         - src: "tests/templates/bifrost-dib-overrides.j2"
           dest: /etc/kolla/config/bifrost/dib.yml
           when: scenario == "bifrost"
+        - src: "tests/templates/ironic-overrides.j2"
+          dest: /etc/kolla/config/ironic.conf
+          when: "{{ scenario == 'ironic' }}"
       when: item.when | default(true)
 
+    - block:
+        - name: ensure ironic config directory exists
+          file:
+            path: /etc/kolla/config/ironic
+            state: directory
+            mode: 0777
+
+        - name: download Ironic Python Agent (IPA) images
+          get_url:
+            url: "https://tarballs.openstack.org/ironic-python-agent/tinyipa/files/{{ item.src }}"
+            dest: "/etc/kolla/config/ironic/{{ item.dest }}"
+          with_items:
+            - src: "tinyipa-{{ zuul.branch | replace('/', '-') }}.gz"
+              dest: ironic-agent.initramfs
+            - src: "tinyipa-{{ zuul.branch | replace('/', '-') }}.vmlinuz"
+              dest: ironic-agent.kernel
+      when: scenario == "ironic"
+
     - name: install kolla-ansible requirements
       pip:
         requirements: "{{ kolla_ansible_src_dir }}/requirements.txt"
@@ -173,7 +194,7 @@
             chdir: "{{ kolla_ansible_src_dir }}"
           environment:
             ACTION: "{{ scenario }}"
-          when: scenario not in ['scenario_nfv']
+          when: scenario not in ['ironic', 'scenario_nfv']
 
         - name: Run test-scenario-nfv.sh script
           script:
@@ -182,6 +203,13 @@
             chdir: "{{ kolla_ansible_src_dir }}"
           when: scenario == "scenario_nfv"
 
+        - name: Run test-ironic.sh script
+          script:
+            cmd: test-ironic.sh
+            executable: /bin/bash
+            chdir: "{{ kolla_ansible_src_dir }}"
+          when: scenario == "ironic"
+
         - name: Run reconfigure.sh script
           script:
             cmd: reconfigure.sh

tests/templates/globals-default.j2

@@ -67,3 +67,8 @@ enable_mistral: "yes"
 enable_redis: "yes"
 enable_barbican: "yes"
 {% endif %}
+
+{% if scenario == "ironic" %}
+enable_ironic: "yes"
+ironic_dnsmasq_dhcp_range: "10.42.0.2,10.42.0.254"
+{% endif %}

tests/templates/ironic-overrides.j2

@@ -0,0 +1,19 @@
+[DEFAULT]
+# Enable all fake hardware types and interfaces.
+enabled_hardware_types = fake-hardware
+enabled_boot_interfaces = fake
+enabled_console_interfaces = ipmitool-socat,no-console
+enabled_deploy_interfaces = fake
+enabled_inspect_interfaces = inspector,no-inspect
+enabled_management_interfaces = fake
+enabled_network_interfaces = noop,flat,neutron
+default_network_interface = neutron
+enabled_power_interfaces = fake
+enabled_raid_interfaces = agent,no-raid
+default_raid_interface = no-raid
+enabled_rescue_interfaces = fake
+enabled_vendor_interfaces = no-vendor
+
+[neutron]
+cleaning_network = public1
+provisioning_network = public1

tests/test-ironic.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+set -o pipefail
+
+# Enable unbuffered output for Ansible in Jenkins.
+export PYTHONUNBUFFERED=1
+
+# Adapted from the function of the same name in the ironic devstack plugin.
+function wait_for_placement_resources {
+    # After nodes have been enrolled, we need to wait for both ironic and
+    # nova's periodic tasks to populate the resource tracker with available
+    # nodes and resources. Wait up to 2 minutes for a given resource before
+    # timing out.
+    local expected_count=1
+    local resource_class="RC0"
+
+    curl -L -o jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
+    chmod +x jq
+
+    # TODO(mgoddard): switch to Placement OSC plugin, once it exists
+    local token
+    token=$(openstack token issue -f value -c id)
+    local endpoint
+    endpoint=$(openstack endpoint list --service placement --interface public -f value -c URL)
+    if [[ -z $endpoint ]]; then
+        echo "Cannot find Placement API endpoint"
+        return 1
+    fi
+
+    local i
+    local count
+    echo "Waiting 2 minutes for Nova resource tracker to pick up $expected_count nodes"
+    for i in $(seq 1 120); do
+        # Fetch provider UUIDs from Placement
+        local providers
+        providers=$(curl -sH "X-Auth-Token: $token" $endpoint/resource_providers \
+            | ./jq -r '.resource_providers[].uuid')
+
+        local p
+        # Total count of the resource class, has to be equal to nodes count
+        count=0
+        for p in $providers; do
+            local amount
+            # A resource class inventory record looks something like
+            # {"max_unit": 1, "min_unit": 1, "step_size": 1, "reserved": 0, "total": 1, "allocation_ratio": 1}
+            # Subtract reserved from total (defaulting both to 0)
+            amount=$(curl -sH "X-Auth-Token: $token" $endpoint/resource_providers/$p/inventories \
+                | ./jq ".inventories.CUSTOM_$resource_class as \$cls
+                    | (\$cls.total // 0) - (\$cls.reserved // 0)")
+            if [ $amount -gt 0 ]; then
+                count=$(( count + $amount ))
+            fi
+        done
+
+        if [ $count -ge $expected_count ]; then
+            return 0
+        fi
+        sleep 1
+    done
+
+    echo "Timed out waiting for Nova to track $expected_count nodes"
+    return 1
+}
+
+function create_resources {
+    # Create a bare metal node and port.
+    openstack baremetal node create \
+        --name node-0 \
+        --driver fake-hardware \
+        --network-interface noop \
+        --property cpu_arch=x86_64 \
+        --resource-class rc0
+    node_uuid=$(openstack baremetal node show node-0 -f value -c uuid)
+    openstack baremetal port create \
+        00:11:22:33:44:55 \
+        --node $node_uuid
+    openstack baremetal node power off node-0
+    openstack baremetal node manage node-0 --wait
+    openstack baremetal node provide node-0 --wait
+
+    # Create a bare metal flavor in nova.
+    openstack flavor create \
+        baremetal \
+        --vcpus 1 \
+        --ram 1024 \
+        --disk 10 \
+        --property resources:CUSTOM_RC0=1 \
+        --property resources:VCPU=0 \
+        --property resources:MEMORY_MB=0 \
+        --property resources:DISK_GB=0 \
+        --public
+}
+
+function test_ironic_logged {
+    # Assumes init-runonce has been executed.
+    . /etc/kolla/admin-openrc.sh
+    . ~/ironic-venv/bin/activate
+
+    # Smoke test ironic API.
+    if ! openstack baremetal driver list | grep fake-hardware; then
+        echo "No active conductors with fake-hardware driver"
+        exit 1
+    fi
+    openstack baremetal node list
+    openstack baremetal port list
+
+    create_resources
+    wait_for_placement_resources
+
+    echo "TESTING: Server creation"
+    openstack server create --wait --image cirros --flavor baremetal --key-name mykey --network demo-net kolla_boot_test
+    openstack --debug server list
+    # If the status is not ACTIVE, print info and exit 1
+    if [[ $(openstack server show kolla_boot_test -f value -c status) != "ACTIVE" ]]; then
+        echo "FAILED: Instance is not active"
+        openstack --debug server show kolla_boot_test
+        return 1
+    fi
+    echo "SUCCESS: Server creation"
+
+    echo "TESTING: Server deletion"
+    openstack server delete --wait kolla_boot_test
+    echo "SUCCESS: Server deletion"
+}
+
+function test_ironic {
+    echo "Testing Ironic"
+    if ! test_ironic_logged > /tmp/logs/ansible/test-ironic 2>&1; then
+        echo "Testing Ironic failed. See ansible/test-ironic for details"
+        return 1
+    else
+        echo "Successfully tested Ironic. See ansible/test-ironic for details"
+    fi
+}
+
+test_ironic

tests/test-openstack.sh

@@ -6,7 +6,6 @@ set -o errexit
 # Enable unbuffered output for Ansible in Jenkins.
 export PYTHONUNBUFFERED=1
 
-
 function test_openstack_logged {
     . /etc/kolla/admin-openrc.sh
 

tools/setup_gate.sh

@@ -42,6 +42,9 @@ EOF
     if [[ $ACTION == "scenario_nfv" ]]; then
         GATE_IMAGES+=",tacker,mistral,redis,barbican"
     fi
+    if [[ $ACTION == "ironic" ]]; then
+        GATE_IMAGES+=",dnsmasq,ironic,iscsid"
+    fi
 
     cat <<EOF | sudo tee /etc/kolla/kolla-build.conf
 [DEFAULT]
@@ -93,6 +96,13 @@ function setup_ansible {
     if [[ $ACTION == "zun" ]]; then
         sudo -H pip install -U "python-zunclient"
     fi
+    if [[ $ACTION == ironic ]]; then
+        # NOTE(mgoddard): Installing python-ironicclient to site-packages fails
+        # due to pip 10 distutils issue with ipaddress package.
+        virtualenv ~/ironic-venv
+        ~/ironic-venv/bin/pip install -U pip
+        ~/ironic-venv/bin/pip install python-openstackclient python-ironicclient
+    fi
     detect_distro
 
     sudo mkdir /etc/ansible
@@ -121,7 +131,6 @@ function prepare_images {
     popd
 }
 
-
 setup_ansible
 setup_config
 setup_node

zuul.d/base.yaml

@@ -37,3 +37,13 @@
       scenario: bifrost
       install_type: source
       enable_core_openstack: no
+
+- job:
+    name: kolla-ansible-ironic-base
+    parent: kolla-ansible-base
+    voting: false
+    files:
+      - ^ansible\/roles\/ironic\/.*
+      - ^tests/test-ironic.sh
+    vars:
+      scenario: ironic

zuul.d/jobs.yaml

@@ -205,3 +205,37 @@
       base_distro: centos
       install_type: source
       scenario: scenario_nfv
+
+- job:
+    name: kolla-ansible-centos-source-ironic
+    parent: kolla-ansible-ironic-base
+    nodeset: kolla-ansible-centos
+    vars:
+      base_distro: centos
+      install_type: source
+
+- job:
+    name: kolla-ansible-centos-binary-ironic
+    parent: kolla-ansible-ironic-base
+    nodeset: kolla-ansible-centos
+    vars:
+      base_distro: centos
+      install_type: binary
+
+- job:
+    name: kolla-ansible-ubuntu-source-ironic
+    parent: kolla-ansible-ironic-base
+    nodeset: kolla-ansible-bionic
+    vars:
+      base_distro: ubuntu
+      install_type: source
+
+# Not running this job because the ironic-neutron-agent image is not available
+# for Ubuntu binary.
+- job:
+    name: kolla-ansible-ubuntu-binary-ironic
+    parent: kolla-ansible-ironic-base
+    nodeset: kolla-ansible-bionic
+    vars:
+      base_distro: ubuntu
+      install_type: binary

zuul.d/project.yaml

@@ -27,6 +27,9 @@
             files: ^ansible\/roles\/(zun|kuryr)\/.*
         - kolla-ansible-centos-source-scenario-nfv:
             files: ^ansible\/roles\/(barbican|heat|mistral|redis|tacker)\/.*
+        - kolla-ansible-centos-source-ironic
+        - kolla-ansible-centos-binary-ironic
+        - kolla-ansible-ubuntu-source-ironic
         - kolla-ansible-centos-source-upgrade
         - kolla-ansible-ubuntu-source-upgrade
         - kolla-ansible-centos-source-upgrade-ceph
@@ -46,6 +49,9 @@
         - kolla-ansible-centos-source-scenario-nfv
         - kolla-ansible-ubuntu-source-cinder-lvm
         - kolla-ansible-centos-source-cinder-lvm
+        - kolla-ansible-centos-source-ironic
+        - kolla-ansible-centos-binary-ironic
+        - kolla-ansible-ubuntu-source-ironic
         - kolla-ansible-centos-source-upgrade
         - kolla-ansible-ubuntu-source-upgrade
         - kolla-ansible-centos-source-upgrade-ceph