diff --git a/tools/deployment/common/force-cronjob-run.sh b/tools/deployment/common/force-cronjob-run.sh new file mode 100755 index 0000000000..b7f2eb69f6 --- /dev/null +++ b/tools/deployment/common/force-cronjob-run.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -e + +TEST_POSTFIX="cronjob-test-run" + +CRONJOBS=$(kubectl get cj --no-headers --all-namespaces | awk '{print $1":"$2}') +for i in ${CRONJOBS}; do + NS=$(echo "$i" | cut -f1 -d:) + CJ=$(echo "$i" | cut -f2 -d:) + + # avoid scheduled runs to prevent case when our manual job is deleted by history limit. + kubectl patch cj/"${CJ}" -n"${NS}" -p '{"spec" : {"suspend" : true }}' + kubectl create job -n"${NS}" --from=cj/"${CJ}" "${CJ}-${TEST_POSTFIX}" +done + +echo "Waiting for all test jobs to complete." +for i in {1..30}; do + RUNNING_JOBS=$(kubectl get jobs --all-namespaces | grep "${TEST_POSTFIX}.*0/1" ||:) + [ -z "${RUNNING_JOBS}" ] && break + sleep 10 +done + +if [ -n "${RUNNING_JOBS}" ]; then + echo -e "Timed out waiting for cj jobs to complete:\n${RUNNING_JOBS}" + exit 1 +fi + +echo "All test jobs completed." + +NOT_FOUND_JOBS="" +echo "Checking that every cronjob has a corresponding test job" +for i in ${CRONJOBS}; do + NS=$(echo "$i" | cut -f1 -d:) + CJ=$(echo "$i" | cut -f2 -d:) + if [ -z "$(kubectl get job -n ${NS} ${CJ}-${TEST_POSTFIX} | grep '1/1' ||:)" ]; then + NOT_FOUND_JOBS="${NOT_FOUND_JOBS}\n$i" + fi +done + +if [ -n "${NOT_FOUND_JOBS}" ]; then + echo -e "Some cronjobs don't have corresponding test jobs:${NOT_FOUND_JOBS}" + kubectl get jobs --all-namespaces | grep ${TEST_POSTFIX} + exit 1 +fi + +echo "Cronjobs run successfully:" +kubectl get jobs --all-namespaces | grep ${TEST_POSTFIX} + +# Unsuspend cj on success assuming they all were not suspended on start +for i in ${CRONJOBS}; do + NS=$(echo "$i" | cut -f1 -d:) + CJ=$(echo "$i" | cut -f2 -d:) + kubectl patch cj/"${CJ}" -n"${NS}" -p '{"spec" : {"suspend" : false }}' +done diff --git a/zuul.d/jobs-openstack-helm.yaml b/zuul.d/jobs-openstack-helm.yaml index befc10369d..ef2e2bc6d6 100644 --- a/zuul.d/jobs-openstack-helm.yaml +++ b/zuul.d/jobs-openstack-helm.yaml @@ -91,6 +91,8 @@ - ./tools/deployment/component/common/rabbitmq.sh - ./tools/deployment/component/keystone/keystone.sh - ./tools/deployment/component/cinder/cinder.sh + - ./tools/deployment/common/force-cronjob-run.sh + - job: name: openstack-helm-cinder-pike-ubuntu_xenial parent: openstack-helm-cinder @@ -189,6 +191,7 @@ ./tools/deployment/common/run-helm-tests.sh keystone; - ./tools/deployment/developer/common/170-setup-gateway.sh - ./tools/deployment/developer/common/900-use-it.sh + - ./tools/deployment/common/force-cronjob-run.sh - job: name: openstack-helm-compute-kit-pike-ubuntu_xenial @@ -312,6 +315,7 @@ - ./tools/deployment/component/compute-kit/compute-kit.sh - ./tools/deployment/developer/common/170-setup-gateway.sh - ./tools/deployment/developer/common/900-use-it.sh + - ./tools/deployment/common/force-cronjob-run.sh - job: name: openstack-helm-apparmor