From de24f25e5846ba442ed723613289eee550fdbd78 Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Mon, 12 Feb 2018 11:28:54 +1100
Subject: [PATCH] Add native distro test jobs

This implements legacy-dsvm-nodepool-[distro]-src jobs as native jobs.
This seems like the best home for them, as they are run by multiple
externals such as glean and diskimage-builder.

Previously the defaults were set such that with no arguments, the
trusty test ran for nodepool-functional-py35.

To facilitate better templating, this turns all builds off by default.
The nodepool functional test is moved to a base template, and then
nodepool-functional-py35 explicitly builds and boots Xenial now
(trusty doesn't seem very useful).

The check_devstack_plugin.sh script runs after devstack, so needs to
source the stackrc file to pull in the variables about what images are
paused or not.

Additionally, the path for the script is fixed to nodepool so we can
run the job from other projects.

The redhat, ubuntu and suse legacy tests are re-implemented for their
respective builds.

This also highlighted that the opensuse test wasn't actually doing
anything.  This actually adds the configuration to build opensuse-423.

Needed-By: https://review.openstack.org/543270
Needed-By: https://review.openstack.org/543328
Needed-By: https://review.openstack.org/543329
Needed-By: https://review.openstack.org/543330
Change-Id: I203d149a1d63ac8358e1c8b878d9c2bc0ba67c02
---
 .zuul.yaml                                    | 41 ++++++++++++++-
 devstack/files/debs/nodepool                  |  2 +
 devstack/files/rpms/nodepool                  |  2 +
 devstack/plugin.sh                            | 34 +++++++++++++
 devstack/settings                             |  3 +-
 .../nodepool-functional-py35-src/run.yaml     |  2 +-
 tools/check_devstack_plugin.sh                | 50 ++++++++++++-------
 7 files changed, 113 insertions(+), 21 deletions(-)

diff --git a/.zuul.yaml b/.zuul.yaml
index ae05f8f14..b8ec2ff58 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -38,7 +38,7 @@
     run: playbooks/nodepool-functional-py35/run.yaml
 
 - job:
-    name: nodepool-functional-py35-src
+    name: nodepool-functional-py35-src-base
     parent: nodepool-functional-base
     run: playbooks/nodepool-functional-py35-src/run.yaml
     vars:
@@ -50,6 +50,13 @@
       - openstack-infra/shade
       - openstack/diskimage-builder
 
+- job:
+    name: nodepool-functional-py35-src
+    parent: nodepool-functional-py35-src-base
+    vars:
+      devstack_localrc:
+        NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB: false
+
 - job:
     name: nodepool-zuul-functional
     parent: tox
@@ -65,6 +72,38 @@
       zuul_work_dir: "{{ zuul.projects['git.openstack.org/openstack-infra/zuul'].src_dir }}"
       tox_envlist: nodepool
 
+# Distro functional tests.  This is used by projects such as
+# diskimage-builder and glean as live-boot tests
+- job:
+    name: nodepool-functional-py35-redhat-src
+    parent: nodepool-functional-py35-src-base
+    vars:
+      devstack_localrc:
+        NODEPOOL_PAUSE_CENTOS_7_DIB: false
+        NODEPOOL_PAUSE_FEDORA_26_DIB: false
+
+- job:
+    name: nodepool-functional-py35-suse-src
+    parent: nodepool-functional-py35-src-base
+    vars:
+      devstack_localrc:
+        NODEPOOL_PAUSE_OPENSUSE_423_DIB: false
+
+- job:
+    name: nodepool-functional-py35-ubuntu-src
+    parent: nodepool-functional-py35-src-base
+    vars:
+      devstack_localrc:
+        NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB: false
+        NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB: false
+
+- job:
+    name: nodepool-functional-py35-debian-src
+    parent: nodepool-functional-py35-src-base
+    vars:
+      devstack_localrc:
+        NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB: false
+
 - project:
     check:
       jobs:
diff --git a/devstack/files/debs/nodepool b/devstack/files/debs/nodepool
index fe8b87512..e636fd0b3 100644
--- a/devstack/files/debs/nodepool
+++ b/devstack/files/debs/nodepool
@@ -3,3 +3,5 @@ kpartx
 debootstrap
 yum-utils
 zookeeperd
+zypper
+gnupg2
diff --git a/devstack/files/rpms/nodepool b/devstack/files/rpms/nodepool
index f5cec3fbb..00be95d4b 100644
--- a/devstack/files/rpms/nodepool
+++ b/devstack/files/rpms/nodepool
@@ -1,3 +1,5 @@
 qemu-img
 kpartx
 debootstrap
+zypper
+gnupg2
diff --git a/devstack/plugin.sh b/devstack/plugin.sh
index 4688c6d21..6b8012b34 100644
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@@ -204,6 +204,7 @@ EOF
     NODEPOOL_FEDORA_26_MIN_READY=1
     NODEPOOL_UBUNTU_TRUSTY_MIN_READY=1
     NODEPOOL_UBUNTU_XENIAL_MIN_READY=1
+    NODEPOOL_OPENSUSE_423_MIN_READY=1
 
     if $NODEPOOL_PAUSE_CENTOS_7_DIB ; then
        NODEPOOL_CENTOS_7_MIN_READY=0
@@ -223,6 +224,9 @@ EOF
     if $NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB ; then
        NODEPOOL_UBUNTU_XENIAL_MIN_READY=0
     fi
+    if $NODEPOOL_PAUSE_OPENSUSE_423_DIB ; then
+        NODEPOOL_OPENSUSE_423_MIN_READY=0
+    fi
 
     cat > /tmp/nodepool.yaml <<EOF
 # You will need to make and populate this path as necessary,
@@ -248,6 +252,8 @@ labels:
     min-ready: $NODEPOOL_UBUNTU_TRUSTY_MIN_READY
   - name: ubuntu-xenial
     min-ready: $NODEPOOL_UBUNTU_XENIAL_MIN_READY
+  - name: opensuse-423
+    min-ready: $NODEPOOL_OPENSUSE_423_MIN_READY
 
 providers:
   - name: devstack
@@ -270,6 +276,8 @@ providers:
         config-drive: true
       - name: ubuntu-xenial
         config-drive: true
+      - name: opensuse-423
+        config-drive: true
     pools:
       - name: main
         max-servers: 5
@@ -304,6 +312,12 @@ providers:
             min-ram: 512
             flavor-name: 'nodepool'
             console-log: True
+          - name: opensuse-423
+            diskimage: opensuse-423
+            min-ram: 512
+            flavor-name: 'nodepool'
+            console-log: True
+
 
 diskimages:
   - name: centos-7
@@ -441,6 +455,26 @@ diskimages:
       $DIB_GLEAN_INSTALLTYPE
       $DIB_GLEAN_REPOLOCATION
       $DIB_GLEAN_REPOREF
+  - name: opensuse-423
+    pause: $NODEPOOL_PAUSE_OPENSUSE_423_DIB
+    rebuild-age: 86400
+    elements:
+      - opensuse-minimal
+      - vm
+      - simple-init
+      - devuser
+      - openssh-server
+      - nodepool-setup
+    release: '42.3'
+    env-vars:
+      TMPDIR: $NODEPOOL_DIB_BASE_PATH/tmp
+      DIB_CHECKSUM: '1'
+      DIB_IMAGE_CACHE: $NODEPOOL_DIB_BASE_PATH/cache
+      DIB_DEV_USER_AUTHORIZED_KEYS: $NODEPOOL_PUBKEY
+      $DIB_GET_PIP
+      $DIB_GLEAN_INSTALLTYPE
+      $DIB_GLEAN_REPOLOCATION
+      $DIB_GLEAN_REPOREF
 EOF
 
     sudo mv /tmp/nodepool.yaml $NODEPOOL_CONFIG
diff --git a/devstack/settings b/devstack/settings
index 868ad22e2..9cd0588af 100644
--- a/devstack/settings
+++ b/devstack/settings
@@ -11,8 +11,9 @@ NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB=${NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB:-true}
 NODEPOOL_PAUSE_FEDORA_25_DIB=${NODEPOOL_PAUSE_FEDORA_25_DIB:-true}
 NODEPOOL_PAUSE_FEDORA_26_DIB=${NODEPOOL_PAUSE_FEDORA_26_DIB:-true}
 NODEPOOL_PAUSE_UBUNTU_PRECISE_DIB=${NODEPOOL_PAUSE_UBUNTU_PRECISE_DIB:-true}
-NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=${NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB:-false}
+NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=${NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB:-true}
 NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB=${NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB:-true}
+NODEPOOL_PAUSE_OPENSUSE_423_DIB=${NODEPOOL_PAUSE_OPENSUSE_423_DIB:-true}
 
 SHADE_REPO_URL=${SHADE_REPO_URL:-https://git.openstack.org/openstack-infra/shade}
 SHADE_REPO_REF=${SHADE_REPO_REF:-master}
diff --git a/playbooks/nodepool-functional-py35-src/run.yaml b/playbooks/nodepool-functional-py35-src/run.yaml
index 00b9dd590..e8abd978e 100644
--- a/playbooks/nodepool-functional-py35-src/run.yaml
+++ b/playbooks/nodepool-functional-py35-src/run.yaml
@@ -3,4 +3,4 @@
     - name: Run nodepool tests
       command: tools/check_devstack_plugin.sh {{ ansible_user_dir }}/work/logs/nodepool
       args:
-        chdir: '{{ zuul.project.src_dir }}'
+        chdir: "{{ zuul.projects['git.openstack.org/openstack-infra/nodepool'].src_dir }}"
diff --git a/tools/check_devstack_plugin.sh b/tools/check_devstack_plugin.sh
index b13c0040e..2451cb5ca 100755
--- a/tools/check_devstack_plugin.sh
+++ b/tools/check_devstack_plugin.sh
@@ -3,25 +3,32 @@
 # NOTE(ianw): remove this and "/opt/stack/new" path when native only
 # jobs that pass this arg
 LOGDIR=${1:-$WORKSPACE/logs/}
-
 if [[ -d /opt/stack/nodepool-venv ]]; then
-    NODEPOOL_INSTALL=${NODEPOOL_INSTALL:-/opt/stack/nodepool-venv}
+    DS_HOME=/opt/stack
 else
-    NODEPOOL_INSTALL=${NODEPOOL_INSTALL:-/opt/stack/new/nodepool-venv}
+    DS_HOME=/opt/stack/new
 fi
+
+NODEPOOL_INSTALL=${NODEPOOL_INSTALL:-${DS_HOME}/nodepool-venv}
 NODEPOOL_CONFIG=${NODEPOOL_CONFIG:-/etc/nodepool/nodepool.yaml}
 NODEPOOL_SECURE=${NODEPOOL_SECURE:-/etc/nodepool/secure.conf}
 NODEPOOL="$NODEPOOL_INSTALL/bin/nodepool -c $NODEPOOL_CONFIG -s $NODEPOOL_SECURE"
 
-# Flags to control which images we build.
-# NOTE(pabelanger): Be sure to also update devstack/settings if you change the
-# defaults.
-NODEPOOL_PAUSE_CENTOS_7_DIB=${NODEPOOL_PAUSE_CENTOS_7_DIB:-true}
-NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB=${NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB:-true}
-NODEPOOL_PAUSE_FEDORA_25_DIB=${NODEPOOL_PAUSE_FEDORA_25_DIB:-true}
-NODEPOOL_PAUSE_FEDORA_26_DIB=${NODEPOOL_PAUSE_FEDORA_26_DIB:-true}
-NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=${NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB:-false}
-NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB=${NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB:-true}
+# Source stackrc so that we get the variables about enabled images set
+# from the devstack job.  That's the ones we'll wait for below.
+if [[ ! -f ${DS_HOME}/devstack/stackrc ]]; then
+    echo "Can not find enabled images from devstack run!"
+    exit 1
+else
+    source ${DS_HOME}/devstack/stackrc
+fi
+NODEPOOL_PAUSE_CENTOS_7_DIB=${NODEPOOL_PAUSE_CENTOS_7_DIB:-True}
+NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB=${NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB:-True}
+NODEPOOL_PAUSE_FEDORA_25_DIB=${NODEPOOL_PAUSE_FEDORA_25_DIB:-True}
+NODEPOOL_PAUSE_FEDORA_26_DIB=${NODEPOOL_PAUSE_FEDORA_26_DIB:-True}
+NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB=${NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB:-True}
+NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB=${NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB:-True}
+NODEPOOL_PAUSE_OPENSUSE_423_DIB=${NODEPOOL_PAUSE_OPENSUSE_423_DIB:-True}
 
 function waitforimage {
     name=$1
@@ -45,48 +52,55 @@ function waitfornode {
     done
 }
 
-if [ $NODEPOOL_PAUSE_CENTOS_7_DIB = 'false' ]; then
+if [ ${NODEPOOL_PAUSE_CENTOS_7_DIB,,} = 'false' ]; then
     # check that image built
     waitforimage centos-7
     # check image was bootable
     waitfornode centos-7
 fi
 
-if [ $NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB = 'false' ]; then
+if [ ${NODEPOOL_PAUSE_DEBIAN_JESSIE_DIB,,} = 'false' ]; then
     # check that image built
     waitforimage debian-jessie
     # check image was bootable
     waitfornode debian-jessie
 fi
 
-if [ $NODEPOOL_PAUSE_FEDORA_25_DIB = 'false' ]; then
+if [ ${NODEPOOL_PAUSE_FEDORA_25_DIB,,} = 'false' ]; then
     # check that image built
     waitforimage fedora-25
     # check image was bootable
     waitfornode fedora-25
 fi
 
-if [ $NODEPOOL_PAUSE_FEDORA_26_DIB = 'false' ]; then
+if [ ${NODEPOOL_PAUSE_FEDORA_26_DIB,,} = 'false' ]; then
     # check that image built
     waitforimage fedora-26
     # check image was bootable
     waitfornode fedora-26
 fi
 
-if [ $NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB = 'false' ]; then
+if [ ${NODEPOOL_PAUSE_UBUNTU_TRUSTY_DIB,,} = 'false' ]; then
     # check that image built
     waitforimage ubuntu-trusty
     # check image was bootable
     waitfornode ubuntu-trusty
 fi
 
-if [ $NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB = 'false' ]; then
+if [ ${NODEPOOL_PAUSE_UBUNTU_XENIAL_DIB,,} = 'false' ]; then
     # check that image built
     waitforimage ubuntu-xenial
     # check image was bootable
     waitfornode ubuntu-xenial
 fi
 
+if [ ${NODEPOOL_PAUSE_OPENSUSE_423_DIB,,} = 'false' ]; then
+    # check that image built
+    waitforimage opensuse-423
+    # check image was bootable
+    waitfornode opensuse-423
+fi
+
 set -o errexit
 # Show the built nodes
 $NODEPOOL list