chore(gate): moves overrides into common location

This patchset allows for overrides to be set and ingested by the gates
as well as the AIO.

Change-Id: I27b6d476d9f3b360edc03dd84669ca236cbfa337

charts/Makefile

@@ -17,7 +17,7 @@ TASK  := build
 EXCLUDES := playbooks roles doc tests tools logs tmp zuul.d releasenotes
 # FIXME(lamt): Excluding the dex-aio as there are linting errors and should be
 # fixed in follow on patch. Once that's done, this can be removed.
-EXCLUDES += dex-aio
+EXCLUDES += dex-aio loki grafana nfs
 CHARTS := $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.)))
 
 .PHONY: $(EXCLUDES) $(CHARTS)

charts/grafana/values_overrides/grafana.yaml

@@ -75,5 +75,5 @@ datasources:
     - name: Loki
       type: loki
       access: proxy
-      url: http://loki-stack.loki-stack:3100
+      url: http://loki.loki-stack:3100
       version: 1

charts/harbor/values_overrides/storageclass.yaml

@@ -0,0 +1,15 @@
+persistence:
+  persistentVolumeClaim:
+    registry:
+      storageClass: nfs-provisioner
+    chartmuseum:
+      storageClass: nfs-provisioner
+    jobservice:
+      storageClass: nfs-provisioner
+    database:
+      storageClass: nfs-provisioner
+      size: 2Gi
+    redis:
+      storageClass: nfs-provisioner
+    trivy:
+      storageClass: nfs-provisioner

charts/harbor/values_overrides/tls.yaml

@@ -0,0 +1,5 @@
+expose:
+  tls:
+    enabled: false
+internalTLS:
+  enabled: false

charts/nfs/values_overrides/nfs.yaml

@@ -0,0 +1,5 @@
+storage:
+  persistentVolumeClaim:
+    size: 10Gi
+  persistentVolumeClaim:
+    class_name: nfs-provisioner

charts/tekton-dashboard/values_overrides/readonly.yaml

@@ -0,0 +1,3 @@
+config:
+  args:
+    read_only: true

roles/run-script/tasks/main.yaml

@@ -18,3 +18,5 @@
     script_path: "{{ script }}"
   args:
     chdir: "{{ zuul.project.src_dir }}"
+  environment:
+    FEATURE_GATES: "{{ aio_params.feature_gates | default('') }}"

tools/deployment/common/env-variables.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+set -e
+
+export API_ADDR=$(kubectl get endpoints kubernetes -o json | jq -r '.subsets[0].addresses[0].ip')
+export API_PORT=$(kubectl get endpoints kubernetes -o json | jq -r '.subsets[0].ports[0].port')

tools/deployment/common/get-values-overrides.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+#
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+set -e
+
+HELM_CHART="$1"
+
+: "${HELM_CHART_ROOT_PATH:="../charts/charts"}"
+: "${FEATURE_GATES:=""}"
+CHARTS_FEATURE_MIX="${FEATURE_GATES}"
+
+function echoerr () {
+  echo "$@" 1>&2;
+}
+
+function generate_awk_exp_from_mask () {
+  local POSITION=1
+  for VALUE in $@; do
+    [ "${VALUE}" -eq 1 ] && echo -n "print \$${POSITION};"
+    POSITION=$((POSITION+1))
+  done
+  echo -e "\n"
+}
+
+function combination () {
+  POWER=$((2**$#))
+  BITS="$(awk "BEGIN { while (c++ < $#) printf \"0\" }")"
+  while [ "${POWER}" -gt 1 ];do
+    POWER=$((POWER-1))
+    BIN="$(bc <<< "obase=2; ${POWER}")"
+    MASK="$(echo "${BITS}" | sed -e "s/0\{${#BIN}\}$/$BIN/" | grep -o .)"
+    #NOTE: This line is odd, but written to support both BSD and GNU utils
+    awk -v ORS="-" "{$(generate_awk_exp_from_mask "$MASK")}" <<< "$@" | awk 1 | sed 's/-$//'
+  done
+}
+
+function replace_variables() {
+  for key in $(env); do
+    local arr=( $(echo $key | awk -F'=' '{ print $1, $2}') )
+    sed -i "s#%%%REPLACE_${arr[0]}%%%#${arr[1]}#g" $@
+  done
+}
+
+function override_file_args () {
+  OVERRIDE_ARGS=""
+  echoerr "We will attempt to use values-override files with the following paths:"
+  for FILE in $(combination ${1//,/ } | uniq | tac); do
+    FILE_PATH="${HELM_CHART_ROOT_PATH}/${HELM_CHART}/values_overrides/${FILE}.yaml"
+    if [ -f "${FILE_PATH}" ]; then replace_variables ${FILE_PATH}
+      OVERRIDE_ARGS+=" --values=${FILE_PATH} "
+    fi
+      echoerr "${FILE_PATH}"
+  done
+  echo "${OVERRIDE_ARGS}"
+}
+
+echoerr "We are going to deploy the service ${HELM_CHART} using ${CONTAINER_DISTRO_NAME} (${CONTAINER_DISTRO_VERSION}) distribution containers."
+#source ${HELM_CHART_ROOT_PATH}/tools/deployment/common/env-variables.sh
+override_file_args "${CHARTS_FEATURE_MIX}"

tools/deployment/common/wait-for-pods.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+set -e
+
+# From Kolla-Kubernetes, orginal authors Kevin Fox & Serguei Bezverkhi
+# Default wait timeout is 900 seconds
+end=$(date +%s)
+timeout=${2:-900}
+end=$((end + timeout))
+while true; do
+    kubectl get pods --namespace=$1 -o json | jq -r \
+        '.items[].status.phase' | grep Pending > /dev/null && \
+        PENDING="True" || PENDING="False"
+    query='.items[]|select(.status.phase=="Running")'
+    query="$query|.status.containerStatuses[].ready"
+    kubectl get pods --namespace=$1 -o json | jq -r "$query" | \
+        grep false > /dev/null && READY="False" || READY="True"
+    kubectl get jobs --namespace=$1 -o json | jq -r \
+        '.items[] | .spec.completions == .status.succeeded' | \
+        grep false > /dev/null && JOBR="False" || JOBR="True"
+    [ $PENDING == "False" -a $READY == "True" -a $JOBR == "True" ] && \
+        break || true
+    sleep 5
+    now=$(date +%s)
+    if [ $now -gt $end ] ; then
+        echo "Containers failed to start after $timeout seconds"
+        echo
+        kubectl get pods --namespace $1 -o wide
+        echo
+        if [ $PENDING == "True" ] ; then
+            echo "Some pods are in pending state:"
+            kubectl get pods --field-selector=status.phase=Pending -n $1 -o wide
+        fi
+        [ $READY == "False" ] && echo "Some pods are not ready"
+        [ $JOBR == "False" ] && echo "Some jobs have not succeeded"
+        exit -1
+    fi
+done

tools/gate/harbor/200-install.sh

@@ -1,9 +1,17 @@
 #!/bin/bash
 set -eux
 
+: ${EXTRA_HELM_ARGS_HARBOR}:="$(./tools/deployment/common/get-values-overrides.sh harbor)"}
+
 NS="harbor"
 kubectl create ns $NS
-helm upgrade --install harbor ./charts/harbor --namespace $NS
-kubectl wait --for=condition=ready pod --timeout=600s --namespace $NS --all
+helm upgrade --install harbor ./charts/harbor \
+  --namespace $NS \
+  --values=${EXTRA_HELM_ARGS_HARBOR}
+
+./tools/deployment/common/wait-for-pods.sh $NS
+helm status harbor
+
 helm test harbor -n $NS
-kubectl --namespace $NS get pod
+
+#kubectl --namespace $NS get pod

tools/gate/loki/200-install.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -eux
 
+: ${EXTRA_HELM_ARGS_LOKI_STACK}:="$(./tools/deployment/common/get-values-overrides.sh loki)"}
+: ${EXTRA_HELM_ARGS_GRAFANA}:="$(./tools/deployment/common/get-values-overrides.sh grafana)"}
 NS="loki-stack"
 GNS="grafana"
 
@@ -10,11 +12,13 @@ helm repo add grafana https://grafana.github.io/helm-charts
 helm repo update
 
 # install loki-stack with Loki and Promtail from Grafana helm charts repo
-helm upgrade --install loki grafana/loki-stack --namespace $NS -f ./tools/gate/loki/loki-stack-values.yaml
-kubectl wait --for=condition=ready pod --timeout=600s --namespace $NS --all
-kubectl --namespace $NS get pod
+helm upgrade --install loki grafana/loki-stack --namespace $NS $EXTRA_HELM_ARGS_LOKI_STACK
+./tools/deployment/common/wait-for-pods.sh $NS
+
+helm status loki
 
 # install Grafana from Grafana helm charts repo
-helm upgrade --install grafana grafana/grafana --namespace $GNS -f ./tools/gate/loki/grafana-values.yaml
-kubectl wait --for=condition=ready pod --timeout=600s --namespace $GNS --all
-kubectl --namespace $GNS get pod
+helm upgrade --install grafana grafana/grafana --namespace $GNS $EXTRA_HELM_ARGS_GRAFANA
+./tools/deployment/common/wait-for-pods.sh $GNS
+
+helm status grafana

tools/gate/tekton/200-install.sh

@@ -7,9 +7,13 @@ NS="tekton-pipelines"
 kubectl create ns $NS
 
 for ele in tekton-pipelines tekton-triggers tekton-dashboard; do
-  helm upgrade --install $ele ./charts/$ele --namespace $NS
+  EXTRA_HELM_ARGS="$(./tools/deployment/common/get-values-overrides.sh $ele)"
+  helm upgrade --install $ele ./charts/$ele --namespace $NS $EXTRA_HELM_ARGS
 done
 
-kubectl wait --for=condition=ready pod --timeout=120s --namespace $NS --all
+./tools/deployment/common/wait-for-pods.sh $NS
+helm status -n $NS tekton-pipelines
+helm status -n $NS tekton-triggers
+helm status -n $NS tekton-dashboard
 
 kubectl --namespace $NS get pod

tools/utils/aio-setup.sh

@@ -13,7 +13,15 @@ set -ex
 : ${HARBOR_VERSION:="1.5.2"}
 
 # ensures we have git
-sudo -E apt install -y git
+sudo -E apt install -y git bc jq
+
+: ${EXTRA_HELM_ARGS_TEKTON_PIPELINES:="$(./tools/deployment/common/get-values-overrides.sh tekton-pipelines)"}
+: ${EXTRA_HELM_ARGS_TEKTON_TRIGGERS:="$(./tools/deployment/common/get-values-overrides.sh tekton-triggers)"}
+: ${EXTRA_HELM_ARGS_TEKTON_DASHBOARD:="$(./tools/deployment/common/get-values-overrides.sh tekton-dashboard)"}
+: ${EXTRA_HELM_ARGS_HARBOR:="$(./tools/deployment/common/get-values-overrides.sh harbor)"}
+: ${EXTRA_HELM_ARGS_LOKI_STACK:="$(./tools/deployment/common/get-values-overrides.sh loki)"}
+: ${EXTRA_HELM_ARGS_GRAFANA:="$(./tools/deployment/common/get-values-overrides.sh grafana)"}
+: ${EXTRA_HELM_ARGS_NFS:="$(./tools/deployment/common/get-values-overrides.sh nfs)"}
 
 # clones upstream rep
 if [ $CLONE_REPO == "true" ]; then
@@ -46,50 +54,18 @@ kubectl label nodes --all openstack-control-plane=enabled --overwrite
 # deploys nfs
 helm upgrade --install nfs-provisioner osh/nfs-provisioner \
   --namespace=$NFS_NS \
-  --set storage.persistentVolumeClaim.size=10Gi \
-  --set storage.persistentVolumeClaim.class_name=nfs-provisioner \
   ${EXTRA_HELM_ARGS_NFS}
 
 # waits for the pods to get ready
-kubectl wait --for=condition=ready pod --timeout=600s --all -n $NFS_NS
+./tools/deployment/common/wait-for-pods.sh $NFS_NS
 
 # deploys harbor
-tee /tmp/harbor.yaml << EOF
-expose:
-  tls:
-    enabled: false
-internalTLS:
-  enabled: false
-persistence:
-  persistentVolumeClaim:
-    registry:
-      storageClass: nfs-provisioner
-    chartmuseum:
-      storageClass: nfs-provisioner
-    jobservice:
-      storageClass: nfs-provisioner
-    database:
-      storageClass: nfs-provisioner
-      size: 5Gi
-    redis:
-      storageClass: nfs-provisioner
-    trivy:
-      storageClass: nfs-provisioner
-EOF
-
 helm upgrade --install harbor harbor/harbor \
   --namespace=$HARBOR_NS \
-  --values=/tmp/harbor.yaml \
   --version=${HARBOR_VERSION} \
   ${EXTRA_HELM_ARGS_HARBOR}
 
 # deploys tekton
-tee /tmp/dashboard.yaml << EOF
-config:
-  args:
-    read_only: true
-EOF
-
 helm upgrade --install tekton-pipelines ${CHART_ROOT_PATH}/tekton-pipelines \
   --namespace=${TEKTON_NS} \
   ${EXTRA_HELM_ARGS_TEKTON_PIPELINES}
@@ -100,96 +76,22 @@ helm upgrade --install tekton-triggers ${CHART_ROOT_PATH}/tekton-triggers \
 
 helm upgrade --install tekton-dashboard ${CHART_ROOT_PATH}/tekton-dashboard \
   --namespace=${TEKTON_NS} \
-  --values=/tmp/dashboard.yaml \
   ${EXTRA_HELM_ARGS_TEKTON_DASHBOARD}
 
 # waits for the pods to get ready
-kubectl wait --for=condition=ready pod --timeout=600s --all -n ${TEKTON_NS}
-kubectl wait --for=condition=ready pod --timeout=600s --all -n ${HARBOR_NS}
-
-tee /tmp/loki.yaml << EOF
-loki:
-  enabled: true
-  ingress:
-    enabled: true
-    hosts:
-      - host: loki.jarvis.local
-        paths: ["/"]
-    public: true
-    annotations:
-      nginx.ingress.kubernetes.io/configuration-snippet: |
-        more_set_headers "X-Frame-Options: deny";
-        more_set_headers "X-XSS-Protection: 1; mode=block";
-      nginx.ingress.kubernetes.io/rewrite-target: /
-promtail:
-  enabled: true
-grafana:
-  enabled: false
-EOF
-
-tee /tmp/grafana.yaml << EOF
-ingress:
-  enabled: true
-  hosts: ["grafana","grafana.jarvis","grafana.jarvis.svc.cluster.local"]
-  public: true
-  annotations:
-    nginx.ingress.kubernetes.io/configuration-snippet: |
-      more_set_headers "X-Frame-Options: deny";
-      more_set_headers "X-XSS-Protection: 1; mode=block";
-    nginx.ingress.kubernetes.io/rewrite-target: /
-  labels: {}
-  path: /
-  hosts:
-    - grafana-jarvis.domain
-  ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services.
-  extraPaths: []
-  tls: []
-resources:
-  limits:
-    cpu: 100m
-    memory: 128Mi
-  requests:
-    cpu: 100m
-    memory: 128Mi
-persistence:
-  type: pvc
-  enabled: false
-  storageClassName: nfs-provisioner
-  accessModes:
-    - ReadWriteOnce
-  size: 10Gi
-  # annotations: {}
-  finalizers:
-    - kubernetes.io/pvc-protection
-adminUser: admin
-# adminPassword: strongpassword
-admin:
-  existingSecret: ""
-  userKey: admin-user
-  passwordKey: admin-password
-datasources:
-  datasources.yaml:
-    apiVersion: 1
-    datasources:
-    - name: Loki
-      type: loki
-      access: proxy
-      url: http://loki.loki-stack:3100
-      version: 1
-EOF
+./tools/deployment/common/wait-for-pods.sh ${TEKTON_NS}
+./tools/deployment/common/wait-for-pods.sh ${HARBOR_NS}
 
 # install loki-stack with Loki and Promtail from Grafana helm charts repo
 helm upgrade --install loki grafana/loki-stack \
   --namespace=${LOKI_NS} \
-  --values=/tmp/loki.yaml \
   ${EXTRA_HELM_ARGS_LOKI_STACK}
 
-kubectl wait --for=condition=ready pod --timeout=600s --namespace ${LOKI_NS} --all
+./tools/deployment/common/wait-for-pods.sh ${LOKI_NS}
 
 # install Grafana from Grafana helm charts repo
 helm upgrade --install grafana grafana/grafana \
   --namespace=${GRAFANA_NS} \
-  --values=/tmp/grafana.yaml \
   ${EXTRA_HELM_ARGS_GRAFANA}
 
-kubectl wait --for=condition=ready pod --timeout=600s --namespace ${GRAFANA_NS} --all
+./tools/deployment/common/wait-for-pods.sh ${GRAFANA_NS}

zuul.d/jobs.yaml

@@ -26,6 +26,8 @@
     description: Runs Tekton charts
     nodeset: airship-charts-single-node
     vars:
+      aio_params:
+        feature_gates: "tls storageclass nfs harbor readonly loki-stack grafana"
       collect_kubernetes_logs_namespace: tekton-pipelines
       gate_scripts:
         - ./tools/utils/aio-setup.sh