Move API (functional) tests to separate jobs

It becomes increasingly harder to maintain functional tests using the fake
hardware type and integration tests in the same jobs. This change creates
two new jobs (for Python 2 and Python 3) and makes API tests run only in them.

Change-Id: I3570323b22d8d4f19635f26733b3ae92bd0c661d

devstack/lib/ironic

@@ -2484,6 +2484,7 @@ function ironic_configure_tempest {
         iniset $TEMPEST_CONFIG network shared_physical_network True
     fi
 
+    if is_service_enabled glance; then
         local image_uuid
         image_uuid=$(openstack image show $IRONIC_IMAGE_NAME -f value -c id)
         iniset $TEMPEST_CONFIG compute image_ref $image_uuid
@@ -2493,6 +2494,7 @@ function ironic_configure_tempest {
         iniset $TEMPEST_CONFIG baremetal whole_disk_image_ref $image_uuid
         image_uuid=$(openstack image show $IRONIC_PARTITIONED_IMAGE_NAME -f value -c id)
         iniset $TEMPEST_CONFIG baremetal partition_image_ref $image_uuid
+    fi
 
     if [[ "$IRONIC_IPXE_ENABLED" == "True" ]] ; then
         iniset $TEMPEST_CONFIG baremetal whole_disk_image_url "http://$IRONIC_HTTP_SERVER:$IRONIC_HTTP_PORT/${IRONIC_WHOLEDISK_IMAGE_NAME}.img"

playbooks/legacy/tempest-dsvm-ironic-functional-python2/run.yaml

@@ -0,0 +1,113 @@
+- hosts: all
+  name: Legacy functional (API) tests job on Python 2
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-extra-vars
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_SPECS_RAM=384"
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_RAMDISK_TYPE=tinyipa"
+
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-extra-vars
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.api"
+
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-vars-early
+            # use tempest plugin
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"TEMPEST_PLUGINS+=' /opt/stack/new/ironic-tempest-plugin'"
+            export TEMPEST_CONCURRENCY=1
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          export PROJECTS="openstack/ironic $PROJECTS"
+          export PROJECTS="openstack/ironic-lib $PROJECTS"
+          export PROJECTS="openstack/ironic-python-agent $PROJECTS"
+          export PROJECTS="openstack/ironic-tempest-plugin $PROJECTS"
+          export PROJECTS="openstack/python-ironicclient $PROJECTS"
+          export PROJECTS="openstack/pyghmi $PROJECTS"
+          export PROJECTS="openstack/virtualbmc $PROJECTS"
+          export PYTHONUNBUFFERED=true
+          export DEVSTACK_GATE_TEMPEST=1
+          export DEVSTACK_GATE_IRONIC=1
+          export DEVSTACK_GATE_NEUTRON=1
+          export DEVSTACK_GATE_VIRT_DRIVER=ironic
+          export DEVSTACK_GATE_CONFIGDRIVE=1
+          export DEVSTACK_GATE_IRONIC_DRIVER=ipmi
+          # Only keystone is required for the functional tests
+          export OVERRIDE_ENABLED_SERVICES="key,mysql,rabbit,ir-api,ir-cond,tempest"
+          # We do not need testing nodes pre-configured
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_BAREMETAL_BASIC_OPS=False"
+          # There is no neutron, so using noop networks.
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_DEFAULT_NETWORK_INTERFACE=noop"
+
+          export BRANCH_OVERRIDE="{{ zuul.override_checkout | default('default') }}"
+          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
+              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
+          fi
+
+          if [[ "$ZUUL_BRANCH" != "stable/ocata" && "$BRANCH_OVERRIDE" != "stable/ocata" ]]; then
+              export DEVSTACK_GATE_TLSPROXY=1
+          fi
+
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_TEMPEST_WHOLE_DISK_IMAGE=True"
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_EPHEMERAL_DISK=0"
+
+          export DEVSTACK_GATE_IRONIC_BUILD_RAMDISK=0
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_INSPECTOR_BUILD_RAMDISK=False"
+
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_COUNT=1"
+
+          # Ensure the ironic-vars-EARLY file exists
+          touch ironic-vars-early
+          # Pull in the EARLY variables injected by the optional builders
+          source ironic-vars-early
+
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin ironic git://git.openstack.org/openstack/ironic"
+
+          # Ensure the ironic-EXTRA-vars file exists
+          touch ironic-extra-vars
+          # Pull in the EXTRA variables injected by the optional builders
+          source ironic-extra-vars
+
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/legacy/tempest-dsvm-ironic-functional-python3/run.yaml

@@ -0,0 +1,122 @@
+- hosts: all
+  name: Legacy functional (API) tests job on Python 3
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-extra-vars
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_SPECS_RAM=384"
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_RAMDISK_TYPE=tinyipa"
+
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-extra-vars
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.api"
+
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-vars-early
+            # use tempest plugin
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"TEMPEST_PLUGINS+=' /opt/stack/new/ironic-tempest-plugin'"
+            export TEMPEST_CONCURRENCY=1
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-extra-vars
+            export DEVSTACK_GATE_USE_PYTHON3=True
+
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          export PROJECTS="openstack/ironic $PROJECTS"
+          export PROJECTS="openstack/ironic-lib $PROJECTS"
+          export PROJECTS="openstack/ironic-python-agent $PROJECTS"
+          export PROJECTS="openstack/ironic-tempest-plugin $PROJECTS"
+          export PROJECTS="openstack/python-ironicclient $PROJECTS"
+          export PROJECTS="openstack/pyghmi $PROJECTS"
+          export PROJECTS="openstack/virtualbmc $PROJECTS"
+          export PYTHONUNBUFFERED=true
+          export DEVSTACK_GATE_TEMPEST=1
+          export DEVSTACK_GATE_IRONIC=1
+          export DEVSTACK_GATE_NEUTRON=1
+          export DEVSTACK_GATE_VIRT_DRIVER=ironic
+          export DEVSTACK_GATE_CONFIGDRIVE=1
+          export DEVSTACK_GATE_IRONIC_DRIVER=ipmi
+          # Only keystone is required for the functional tests
+          export OVERRIDE_ENABLED_SERVICES="key,mysql,rabbit,ir-api,ir-cond,tempest"
+          # We do not need testing nodes pre-configured
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_BAREMETAL_BASIC_OPS=False"
+          # There is no neutron, so using noop networks.
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_DEFAULT_NETWORK_INTERFACE=noop"
+
+          export BRANCH_OVERRIDE="{{ zuul.override_checkout | default('default') }}"
+          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
+              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
+          fi
+
+          if [[ "$ZUUL_BRANCH" != "stable/ocata" && "$BRANCH_OVERRIDE" != "stable/ocata" ]]; then
+              export DEVSTACK_GATE_TLSPROXY=1
+          fi
+
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_TEMPEST_WHOLE_DISK_IMAGE=False"
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_EPHEMERAL_DISK=1"
+
+          export DEVSTACK_GATE_IRONIC_BUILD_RAMDISK=0
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_INSPECTOR_BUILD_RAMDISK=False"
+
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_COUNT=1"
+
+          # Ensure the ironic-vars-EARLY file exists
+          touch ironic-vars-early
+          # Pull in the EARLY variables injected by the optional builders
+          source ironic-vars-early
+
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin ironic git://git.openstack.org/openstack/ironic"
+
+          # Ensure the ironic-EXTRA-vars file exists
+          touch ironic-extra-vars
+          # Pull in the EXTRA variables injected by the optional builders
+          source ironic-extra-vars
+
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/legacy/tempest-dsvm-ironic-ipa-partition-pxe_ipmitool-tinyipa-python3/run.yaml

@@ -37,7 +37,7 @@
     - shell:
         cmd: |
           cat << 'EOF' >> ironic-extra-vars
-            export DEVSTACK_GATE_TEMPEST_REGEX="ironic"
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'

playbooks/legacy/tempest-dsvm-ironic-ipa-partition-redfish-tinyipa/run.yaml

@@ -37,7 +37,7 @@
     - shell:
         cmd: |
           cat << 'EOF' >> ironic-extra-vars
-            export DEVSTACK_GATE_TEMPEST_REGEX="ironic"
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'

playbooks/legacy/tempest-dsvm-ironic-ipa-partition-uefi-pxe_ipmitool-tinyipa/run.yaml

@@ -37,7 +37,7 @@
     - shell:
         cmd: |
           cat << 'EOF' >> ironic-extra-vars
-            export DEVSTACK_GATE_TEMPEST_REGEX="ironic"
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'

playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-agent_ipmitool-tinyipa-multinode/run.yaml

@@ -79,7 +79,7 @@
             # Run all ironic tests and the multinode smoke test from nova.
             # TODO(dtantsur): add test_attach_interfaces from nova when our
             # devstack plugin creates more than one port per node.
-            export DEVSTACK_GATE_TEMPEST_REGEX="(ironic|test_schedule_to_all_nodes)"
+            export DEVSTACK_GATE_TEMPEST_REGEX="(ironic_tempest_plugin.tests.scenario|test_schedule_to_all_nodes)"
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'
       environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-bios-agent_ipmitool-tinyipa/run.yaml

@@ -37,7 +37,7 @@
     - shell:
         cmd: |
           cat << 'EOF' >> ironic-extra-vars
-            export DEVSTACK_GATE_TEMPEST_REGEX="ironic"
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'

playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-bios-pxe_snmp-tinyipa/run.yaml

@@ -37,7 +37,7 @@
     - shell:
         cmd: |
           cat << 'EOF' >> ironic-extra-vars
-            export DEVSTACK_GATE_TEMPEST_REGEX="ironic"
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'

playbooks/legacy/tempest-dsvm-ironic-parallel/run.yaml

@@ -46,7 +46,7 @@
               export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
           fi
           # Run only baremetal tests
-          export DEVSTACK_GATE_TEMPEST_REGEX='ironic'
+          export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
           ./safe-devstack-vm-gate-wrap.sh

playbooks/legacy/tempest-dsvm-ironic-pxe_ipmitool-postgres/run.yaml

@@ -37,7 +37,7 @@
     - shell:
         cmd: |
           cat << 'EOF' >> ironic-extra-vars
-            export DEVSTACK_GATE_TEMPEST_REGEX="ironic"
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'

zuul.d/legacy-ironic-jobs.yaml

@@ -204,3 +204,15 @@
     parent: legacy-ironic-dsvm-base
     run: playbooks/legacy/tempest-dsvm-ironic-pxe_ipa-full/run.yaml
     timeout: 9600
+
+- job:
+    name: ironic-tempest-dsvm-functional-python2
+    parent: legacy-ironic-dsvm-base
+    run: playbooks/legacy/tempest-dsvm-ironic-functional-python2/run.yaml
+    timeout: 5400
+
+- job:
+    name: ironic-tempest-dsvm-functional-python3
+    parent: legacy-ironic-dsvm-base
+    run: playbooks/legacy/tempest-dsvm-ironic-functional-python3/run.yaml
+    timeout: 5400

zuul.d/project.yaml

@@ -2,6 +2,8 @@
     check:
       jobs:
         - ironic-dsvm-standalone
+        - ironic-tempest-dsvm-functional-python2
+        - ironic-tempest-dsvm-functional-python3
         - ironic-grenade-dsvm
         - ironic-grenade-dsvm-multinode-multitenant
         - ironic-tempest-dsvm-bfv
@@ -22,6 +24,8 @@
       queue: ironic
       jobs:
         - ironic-dsvm-standalone
+        - ironic-tempest-dsvm-functional-python2
+        - ironic-tempest-dsvm-functional-python3
         - ironic-grenade-dsvm
         - ironic-grenade-dsvm-multinode-multitenant
         - ironic-tempest-dsvm-bfv