Various gate fixes to make gate green

1. There is an ocassional timing issue when container logs are
   unavailabile at certain points in the crash loop at the same
   time the gate script tries to request them. The gate will now retry
   this operation, instead of terminating right away with failure.
2. Re-enable uamlite security context so that useradd operations would
   succeed.
3. Change apt pinning tests to use a version of the package that is
   available in the apt repo. Upstream repos change, so we should not
   pin to an explicit version that will be removed in the future and
   break the gate.
4. Update helm version to 2.14.1 to sync with openstack-helm-infra
5. Fix divingbell build script: git --depth=1 incompatible with explicit
   non-master commit checkout
6. Enhance overrides test case #7 to test for the issue identified in
   [0].
7. Change hostname scheduling to match minikube hostname now configured
   by OSH gate, instead of using the node's actual hostname
8. Re-enable gate voting

[0] https://storyboard.openstack.org/#!/story/2005936

Depends-On: https://review.opendev.org/671875/
Change-Id: Iad983ce363711e16ccd54e663c23d30a4a6a1177

.zuul.yaml

@@ -23,19 +23,17 @@
               - ^.*\.rst$
               - ^doc/.*$
               - ^releasenotes/.*$
-            voting: false
         - airship-divingbell-chart-build-gate
         - airship-divingbell-chart-build-latest-htk
     gate:
       jobs:
         - airship-divingbell-linter
         - airship-divingbell-chart-build-gate
-        # NOTE(portdirect): enable when voting enabled.
-        # - airship-divingbell-ubuntu:
-        #     irrelevant-files:
-        #       - ^.*\.rst$
-        #       - ^doc/.*$
-        #       - ^releasenotes/.*$
+        - airship-divingbell-ubuntu:
+            irrelevant-files:
+              - ^.*\.rst$
+              - ^doc/.*$
+              - ^releasenotes/.*$
 
     post:
       jobs:

divingbell/templates/daemonset-uamlite.yaml

@@ -50,6 +50,11 @@ spec:
           mountPath: /tmp/{{ $daemonset }}.sh
           subPath: {{ $daemonset }}
           readOnly: true
+        # adduser function hangs immediately on getent/useradd without security context
+        # TODO: investigate which linux capabilities should be whitelisted so blanket security
+        # context may be removed
+        securityContext:
+          privileged: true
       volumes:
       - name: rootfs-{{ $daemonset }}
         hostPath:

tools/gate/scripts/020-test-divingbell.sh

@@ -48,10 +48,12 @@ USERNAME3_SUDO=true
 USERNAME4=userfour
 USERNAME4_SUDO=false
 APT_PACKAGE1=python-pbr
-APT_VERSION1=1.8.0-4ubuntu1
+# Pick an available version in the package repo
+APT_VERSION1="$(apt-cache show $APT_PACKAGE1 | grep Version: | tail -1 | awk '{print $2}')"
 APT_PACKAGE2=mysql-server
 APT_PACKAGE3=python-simplejson
-APT_VERSION3=3.8.1-1ubuntu2
+# Pick an available version in the package repo
+APT_VERSION3="$(apt-cache show $APT_PACKAGE3 | grep Version: | tail -1 | awk '{print $2}')"
 APT_PACKAGE4=less
 APT_PACKAGE5=python-setuptools
 APT_PACKAGE6=telnetd
@@ -381,7 +383,7 @@ get_container_status(){
   local container_runtime_sleep_interval=5
   wait_time=0
   while : ; do
-    CLOGS="$(kubectl logs --namespace="${NAME}" "${container}" 2>&1)"
+    CLOGS="$(kubectl logs --namespace="${NAME}" "${container}" 2>&1)" || true
     local status="$(echo "${CLOGS}" | tail -1)"
     if [[ $(echo -e ${status} | tr -d '[:cntrl:]') = *ERROR* ]] ||
        [[ $(echo -e ${status} | tr -d '[:cntrl:]') = *TRACE* ]]; then
@@ -1869,9 +1871,21 @@ test_overrides(){
   echo '[SUCCESS] overrides test 6 passed successfully' >> "${TEST_RESULTS}" ||
   (echo '[FAILURE] overrides test 6 failed' && exit 1)
 
+  # The core functional test to ensure that overrides work.
+  # fooKey was added to catch a corner case identified by:
+  # https://storyboard.openstack.org/#!/story/2005936
+  # If fooHost keys are leaking into this host's values, then this test
+  # will fail when sysctl attempts to set the non-existant fooKey.
   overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-functional.yaml
   key1_override_val=0
   key2_non_override_val=0
+  kube_hostname="$(kubectl describe nodes | grep kubernetes.io/hostname | head -1 | cut -d'=' -f2)" || true
+  if [[ -z $kube_hostname ]]; then
+    fallback_kube_hostname=minikube
+    echo "[WARNING] Failed to get kubectl hostname, falling back to default $fallback_kube_hostname"
+    echo "This test will fail if the kubernetes.io/hostname does not map to the node running this instance of k8s."
+    kube_hostname="$fallback_kube_hostname"
+  fi
   echo "conf:
   sysctl:
     $SYSCTL_KEY1: 1
@@ -1879,13 +1893,18 @@ test_overrides(){
   overrides:
     divingbell_sysctl:
       hosts:
-      - name: $(hostname -f)
+      - name: fooHost
+        conf:
+          sysctl:
+            fooKey: fooVal
+      - name: $kube_hostname
         conf:
           sysctl:
             $SYSCTL_KEY1: $key1_override_val" > "${overrides_yaml}"
   install_base "--values=${overrides_yaml}"
   get_container_status sysctl
-  _test_sysctl_default $SYSCTL_KEY1 $key1_override_val
+  _test_sysctl_default $SYSCTL_KEY1 $key1_override_val || \
+  (echo '[FAILURE] overrides test 7 failed, most likely someone broke openstack-helm-infra/helm-toolkit/templates/utils/_daemonset_overrides.tpl' && exit 1)
   _test_sysctl_default $SYSCTL_KEY2 $key2_non_override_val
   echo '[SUCCESS] overrides test 7 passed successfully' >> "${TEST_RESULTS}"
 

tools/helm_install.sh

@@ -17,7 +17,7 @@
 set -x
 
 HELM=$1
-HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz"}
+HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://storage.googleapis.com/kubernetes-helm/helm-v2.14.1-linux-amd64.tar.gz"}
 
 
 function install_helm_binary {

tools/helm_tk.sh

@@ -18,7 +18,7 @@
 HELM=$1
 HTK_REPO=${HTK_REPO:-"https://github.com/openstack/openstack-helm-infra"}
 HTK_PATH=${HTK_PATH:-""}
-HTK_STABLE_COMMIT=${HTK_COMMIT:-"200b5e902b3a176fbfbe669b6a10a254c9b50f5d"}
+HTK_STABLE_COMMIT=${HTK_COMMIT}
 BUILD_DIR=${BUILD_DIR:-$(mktemp -d)}
 DEP_UP_LIST=${DEP_UP_LIST:-"divingbell"}
 
@@ -54,9 +54,11 @@ function helm_serve {
 
 mkdir -p build
 pushd build
-git clone --depth 1 $HTK_REPO || true
+git clone $HTK_REPO || true
 pushd ${HTK_REPO##*/}/$HTK_PATH
-git reset --hard "${HTK_STABLE_COMMIT}"
+if [[ -n ${HTK_STABLE_COMMIT} ]]; then
+  git reset --hard "${HTK_STABLE_COMMIT}"
+fi
 
 helm_serve
 if [[ ${HELM} != "helm" ]]