From d9b7c293e1a41098a5ef13ff3340e354c55244e8 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 24 Dec 2020 16:40:09 +0000 Subject: [PATCH] chore(gate): moves overrides into common location This patchset allows for overrides to be set and ingested by the gates as well as the AIO. Change-Id: I27b6d476d9f3b360edc03dd84669ca236cbfa337 --- charts/Makefile | 2 +- .../grafana/values_overrides/grafana.yaml | 2 +- .../harbor/values_overrides/storageclass.yaml | 15 +++ charts/harbor/values_overrides/tls.yaml | 5 + .../loki/values_overrides/loki-stack.yaml | 0 charts/nfs/values_overrides/nfs.yaml | 5 + .../values_overrides/readonly.yaml | 3 + roles/run-script/tasks/main.yaml | 2 + tools/deployment/common/env-variables.sh | 17 +++ .../deployment/common/get-values-overrides.sh | 70 ++++++++++ tools/deployment/common/wait-for-pods.sh | 49 +++++++ tools/gate/harbor/200-install.sh | 14 +- tools/gate/loki/200-install.sh | 16 ++- tools/gate/tekton/200-install.sh | 8 +- tools/utils/aio-setup.sh | 126 ++---------------- zuul.d/jobs.yaml | 2 + 16 files changed, 211 insertions(+), 125 deletions(-) rename tools/gate/loki/grafana-values.yaml => charts/grafana/values_overrides/grafana.yaml (97%) create mode 100644 charts/harbor/values_overrides/storageclass.yaml create mode 100644 charts/harbor/values_overrides/tls.yaml rename tools/gate/loki/loki-stack-values.yaml => charts/loki/values_overrides/loki-stack.yaml (100%) create mode 100644 charts/nfs/values_overrides/nfs.yaml create mode 100644 charts/tekton-dashboard/values_overrides/readonly.yaml create mode 100755 tools/deployment/common/env-variables.sh create mode 100755 tools/deployment/common/get-values-overrides.sh create mode 100755 tools/deployment/common/wait-for-pods.sh diff --git a/charts/Makefile b/charts/Makefile index 41fb1c94..d71fa28a 100644 --- a/charts/Makefile +++ b/charts/Makefile @@ -17,7 +17,7 @@ TASK := build EXCLUDES := playbooks roles doc tests tools logs tmp zuul.d releasenotes # FIXME(lamt): Excluding the dex-aio as there are linting errors and should be # fixed in follow on patch. Once that's done, this can be removed. -EXCLUDES += dex-aio +EXCLUDES += dex-aio loki grafana nfs CHARTS := $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.))) .PHONY: $(EXCLUDES) $(CHARTS) diff --git a/tools/gate/loki/grafana-values.yaml b/charts/grafana/values_overrides/grafana.yaml similarity index 97% rename from tools/gate/loki/grafana-values.yaml rename to charts/grafana/values_overrides/grafana.yaml index f29a7a21..dc588703 100644 --- a/tools/gate/loki/grafana-values.yaml +++ b/charts/grafana/values_overrides/grafana.yaml @@ -75,5 +75,5 @@ datasources: - name: Loki type: loki access: proxy - url: http://loki-stack.loki-stack:3100 + url: http://loki.loki-stack:3100 version: 1 diff --git a/charts/harbor/values_overrides/storageclass.yaml b/charts/harbor/values_overrides/storageclass.yaml new file mode 100644 index 00000000..e26eadee --- /dev/null +++ b/charts/harbor/values_overrides/storageclass.yaml @@ -0,0 +1,15 @@ +persistence: + persistentVolumeClaim: + registry: + storageClass: nfs-provisioner + chartmuseum: + storageClass: nfs-provisioner + jobservice: + storageClass: nfs-provisioner + database: + storageClass: nfs-provisioner + size: 2Gi + redis: + storageClass: nfs-provisioner + trivy: + storageClass: nfs-provisioner diff --git a/charts/harbor/values_overrides/tls.yaml b/charts/harbor/values_overrides/tls.yaml new file mode 100644 index 00000000..c02f3d84 --- /dev/null +++ b/charts/harbor/values_overrides/tls.yaml @@ -0,0 +1,5 @@ +expose: + tls: + enabled: false +internalTLS: + enabled: false diff --git a/tools/gate/loki/loki-stack-values.yaml b/charts/loki/values_overrides/loki-stack.yaml similarity index 100% rename from tools/gate/loki/loki-stack-values.yaml rename to charts/loki/values_overrides/loki-stack.yaml diff --git a/charts/nfs/values_overrides/nfs.yaml b/charts/nfs/values_overrides/nfs.yaml new file mode 100644 index 00000000..c203ab6b --- /dev/null +++ b/charts/nfs/values_overrides/nfs.yaml @@ -0,0 +1,5 @@ +storage: + persistentVolumeClaim: + size: 10Gi + persistentVolumeClaim: + class_name: nfs-provisioner \ No newline at end of file diff --git a/charts/tekton-dashboard/values_overrides/readonly.yaml b/charts/tekton-dashboard/values_overrides/readonly.yaml new file mode 100644 index 00000000..cb9aef06 --- /dev/null +++ b/charts/tekton-dashboard/values_overrides/readonly.yaml @@ -0,0 +1,3 @@ +config: + args: + read_only: true \ No newline at end of file diff --git a/roles/run-script/tasks/main.yaml b/roles/run-script/tasks/main.yaml index 463cf205..ffd2b7bc 100644 --- a/roles/run-script/tasks/main.yaml +++ b/roles/run-script/tasks/main.yaml @@ -18,3 +18,5 @@ script_path: "{{ script }}" args: chdir: "{{ zuul.project.src_dir }}" + environment: + FEATURE_GATES: "{{ aio_params.feature_gates | default('') }}" diff --git a/tools/deployment/common/env-variables.sh b/tools/deployment/common/env-variables.sh new file mode 100755 index 00000000..f4f407f8 --- /dev/null +++ b/tools/deployment/common/env-variables.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -e + +export API_ADDR=$(kubectl get endpoints kubernetes -o json | jq -r '.subsets[0].addresses[0].ip') +export API_PORT=$(kubectl get endpoints kubernetes -o json | jq -r '.subsets[0].ports[0].port') diff --git a/tools/deployment/common/get-values-overrides.sh b/tools/deployment/common/get-values-overrides.sh new file mode 100755 index 00000000..da8b1a45 --- /dev/null +++ b/tools/deployment/common/get-values-overrides.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -e + +HELM_CHART="$1" + +: "${HELM_CHART_ROOT_PATH:="../charts/charts"}" +: "${FEATURE_GATES:=""}" +CHARTS_FEATURE_MIX="${FEATURE_GATES}" + +function echoerr () { + echo "$@" 1>&2; +} + +function generate_awk_exp_from_mask () { + local POSITION=1 + for VALUE in $@; do + [ "${VALUE}" -eq 1 ] && echo -n "print \$${POSITION};" + POSITION=$((POSITION+1)) + done + echo -e "\n" +} + +function combination () { + POWER=$((2**$#)) + BITS="$(awk "BEGIN { while (c++ < $#) printf \"0\" }")" + while [ "${POWER}" -gt 1 ];do + POWER=$((POWER-1)) + BIN="$(bc <<< "obase=2; ${POWER}")" + MASK="$(echo "${BITS}" | sed -e "s/0\{${#BIN}\}$/$BIN/" | grep -o .)" + #NOTE: This line is odd, but written to support both BSD and GNU utils + awk -v ORS="-" "{$(generate_awk_exp_from_mask "$MASK")}" <<< "$@" | awk 1 | sed 's/-$//' + done +} + +function replace_variables() { + for key in $(env); do + local arr=( $(echo $key | awk -F'=' '{ print $1, $2}') ) + sed -i "s#%%%REPLACE_${arr[0]}%%%#${arr[1]}#g" $@ + done +} + +function override_file_args () { + OVERRIDE_ARGS="" + echoerr "We will attempt to use values-override files with the following paths:" + for FILE in $(combination ${1//,/ } | uniq | tac); do + FILE_PATH="${HELM_CHART_ROOT_PATH}/${HELM_CHART}/values_overrides/${FILE}.yaml" + if [ -f "${FILE_PATH}" ]; then replace_variables ${FILE_PATH} + OVERRIDE_ARGS+=" --values=${FILE_PATH} " + fi + echoerr "${FILE_PATH}" + done + echo "${OVERRIDE_ARGS}" +} + +echoerr "We are going to deploy the service ${HELM_CHART} using ${CONTAINER_DISTRO_NAME} (${CONTAINER_DISTRO_VERSION}) distribution containers." +#source ${HELM_CHART_ROOT_PATH}/tools/deployment/common/env-variables.sh +override_file_args "${CHARTS_FEATURE_MIX}" diff --git a/tools/deployment/common/wait-for-pods.sh b/tools/deployment/common/wait-for-pods.sh new file mode 100755 index 00000000..5930fcb7 --- /dev/null +++ b/tools/deployment/common/wait-for-pods.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -e + +# From Kolla-Kubernetes, orginal authors Kevin Fox & Serguei Bezverkhi +# Default wait timeout is 900 seconds +end=$(date +%s) +timeout=${2:-900} +end=$((end + timeout)) +while true; do + kubectl get pods --namespace=$1 -o json | jq -r \ + '.items[].status.phase' | grep Pending > /dev/null && \ + PENDING="True" || PENDING="False" + query='.items[]|select(.status.phase=="Running")' + query="$query|.status.containerStatuses[].ready" + kubectl get pods --namespace=$1 -o json | jq -r "$query" | \ + grep false > /dev/null && READY="False" || READY="True" + kubectl get jobs --namespace=$1 -o json | jq -r \ + '.items[] | .spec.completions == .status.succeeded' | \ + grep false > /dev/null && JOBR="False" || JOBR="True" + [ $PENDING == "False" -a $READY == "True" -a $JOBR == "True" ] && \ + break || true + sleep 5 + now=$(date +%s) + if [ $now -gt $end ] ; then + echo "Containers failed to start after $timeout seconds" + echo + kubectl get pods --namespace $1 -o wide + echo + if [ $PENDING == "True" ] ; then + echo "Some pods are in pending state:" + kubectl get pods --field-selector=status.phase=Pending -n $1 -o wide + fi + [ $READY == "False" ] && echo "Some pods are not ready" + [ $JOBR == "False" ] && echo "Some jobs have not succeeded" + exit -1 + fi +done diff --git a/tools/gate/harbor/200-install.sh b/tools/gate/harbor/200-install.sh index 7e2cdf69..da238b8f 100755 --- a/tools/gate/harbor/200-install.sh +++ b/tools/gate/harbor/200-install.sh @@ -1,9 +1,17 @@ #!/bin/bash set -eux +: ${EXTRA_HELM_ARGS_HARBOR}:="$(./tools/deployment/common/get-values-overrides.sh harbor)"} + NS="harbor" kubectl create ns $NS -helm upgrade --install harbor ./charts/harbor --namespace $NS -kubectl wait --for=condition=ready pod --timeout=600s --namespace $NS --all +helm upgrade --install harbor ./charts/harbor \ + --namespace $NS \ + --values=${EXTRA_HELM_ARGS_HARBOR} + +./tools/deployment/common/wait-for-pods.sh $NS +helm status harbor + helm test harbor -n $NS -kubectl --namespace $NS get pod + +#kubectl --namespace $NS get pod diff --git a/tools/gate/loki/200-install.sh b/tools/gate/loki/200-install.sh index 7836ad1b..6d097204 100755 --- a/tools/gate/loki/200-install.sh +++ b/tools/gate/loki/200-install.sh @@ -1,6 +1,8 @@ #!/bin/bash set -eux +: ${EXTRA_HELM_ARGS_LOKI_STACK}:="$(./tools/deployment/common/get-values-overrides.sh loki)"} +: ${EXTRA_HELM_ARGS_GRAFANA}:="$(./tools/deployment/common/get-values-overrides.sh grafana)"} NS="loki-stack" GNS="grafana" @@ -10,11 +12,13 @@ helm repo add grafana https://grafana.github.io/helm-charts helm repo update # install loki-stack with Loki and Promtail from Grafana helm charts repo -helm upgrade --install loki grafana/loki-stack --namespace $NS -f ./tools/gate/loki/loki-stack-values.yaml -kubectl wait --for=condition=ready pod --timeout=600s --namespace $NS --all -kubectl --namespace $NS get pod +helm upgrade --install loki grafana/loki-stack --namespace $NS $EXTRA_HELM_ARGS_LOKI_STACK +./tools/deployment/common/wait-for-pods.sh $NS + +helm status loki # install Grafana from Grafana helm charts repo -helm upgrade --install grafana grafana/grafana --namespace $GNS -f ./tools/gate/loki/grafana-values.yaml -kubectl wait --for=condition=ready pod --timeout=600s --namespace $GNS --all -kubectl --namespace $GNS get pod +helm upgrade --install grafana grafana/grafana --namespace $GNS $EXTRA_HELM_ARGS_GRAFANA +./tools/deployment/common/wait-for-pods.sh $GNS + +helm status grafana \ No newline at end of file diff --git a/tools/gate/tekton/200-install.sh b/tools/gate/tekton/200-install.sh index 155a45db..d32f8890 100755 --- a/tools/gate/tekton/200-install.sh +++ b/tools/gate/tekton/200-install.sh @@ -7,9 +7,13 @@ NS="tekton-pipelines" kubectl create ns $NS for ele in tekton-pipelines tekton-triggers tekton-dashboard; do - helm upgrade --install $ele ./charts/$ele --namespace $NS + EXTRA_HELM_ARGS="$(./tools/deployment/common/get-values-overrides.sh $ele)" + helm upgrade --install $ele ./charts/$ele --namespace $NS $EXTRA_HELM_ARGS done -kubectl wait --for=condition=ready pod --timeout=120s --namespace $NS --all +./tools/deployment/common/wait-for-pods.sh $NS +helm status -n $NS tekton-pipelines +helm status -n $NS tekton-triggers +helm status -n $NS tekton-dashboard kubectl --namespace $NS get pod diff --git a/tools/utils/aio-setup.sh b/tools/utils/aio-setup.sh index 1439b1b8..7a700096 100755 --- a/tools/utils/aio-setup.sh +++ b/tools/utils/aio-setup.sh @@ -13,7 +13,15 @@ set -ex : ${HARBOR_VERSION:="1.5.2"} # ensures we have git -sudo -E apt install -y git +sudo -E apt install -y git bc jq + +: ${EXTRA_HELM_ARGS_TEKTON_PIPELINES:="$(./tools/deployment/common/get-values-overrides.sh tekton-pipelines)"} +: ${EXTRA_HELM_ARGS_TEKTON_TRIGGERS:="$(./tools/deployment/common/get-values-overrides.sh tekton-triggers)"} +: ${EXTRA_HELM_ARGS_TEKTON_DASHBOARD:="$(./tools/deployment/common/get-values-overrides.sh tekton-dashboard)"} +: ${EXTRA_HELM_ARGS_HARBOR:="$(./tools/deployment/common/get-values-overrides.sh harbor)"} +: ${EXTRA_HELM_ARGS_LOKI_STACK:="$(./tools/deployment/common/get-values-overrides.sh loki)"} +: ${EXTRA_HELM_ARGS_GRAFANA:="$(./tools/deployment/common/get-values-overrides.sh grafana)"} +: ${EXTRA_HELM_ARGS_NFS:="$(./tools/deployment/common/get-values-overrides.sh nfs)"} # clones upstream rep if [ $CLONE_REPO == "true" ]; then @@ -46,50 +54,18 @@ kubectl label nodes --all openstack-control-plane=enabled --overwrite # deploys nfs helm upgrade --install nfs-provisioner osh/nfs-provisioner \ --namespace=$NFS_NS \ - --set storage.persistentVolumeClaim.size=10Gi \ - --set storage.persistentVolumeClaim.class_name=nfs-provisioner \ ${EXTRA_HELM_ARGS_NFS} # waits for the pods to get ready -kubectl wait --for=condition=ready pod --timeout=600s --all -n $NFS_NS +./tools/deployment/common/wait-for-pods.sh $NFS_NS # deploys harbor -tee /tmp/harbor.yaml << EOF -expose: - tls: - enabled: false -internalTLS: - enabled: false -persistence: - persistentVolumeClaim: - registry: - storageClass: nfs-provisioner - chartmuseum: - storageClass: nfs-provisioner - jobservice: - storageClass: nfs-provisioner - database: - storageClass: nfs-provisioner - size: 5Gi - redis: - storageClass: nfs-provisioner - trivy: - storageClass: nfs-provisioner -EOF - helm upgrade --install harbor harbor/harbor \ --namespace=$HARBOR_NS \ - --values=/tmp/harbor.yaml \ --version=${HARBOR_VERSION} \ ${EXTRA_HELM_ARGS_HARBOR} # deploys tekton -tee /tmp/dashboard.yaml << EOF -config: - args: - read_only: true -EOF - helm upgrade --install tekton-pipelines ${CHART_ROOT_PATH}/tekton-pipelines \ --namespace=${TEKTON_NS} \ ${EXTRA_HELM_ARGS_TEKTON_PIPELINES} @@ -100,96 +76,22 @@ helm upgrade --install tekton-triggers ${CHART_ROOT_PATH}/tekton-triggers \ helm upgrade --install tekton-dashboard ${CHART_ROOT_PATH}/tekton-dashboard \ --namespace=${TEKTON_NS} \ - --values=/tmp/dashboard.yaml \ ${EXTRA_HELM_ARGS_TEKTON_DASHBOARD} # waits for the pods to get ready -kubectl wait --for=condition=ready pod --timeout=600s --all -n ${TEKTON_NS} -kubectl wait --for=condition=ready pod --timeout=600s --all -n ${HARBOR_NS} - -tee /tmp/loki.yaml << EOF -loki: - enabled: true - ingress: - enabled: true - hosts: - - host: loki.jarvis.local - paths: ["/"] - public: true - annotations: - nginx.ingress.kubernetes.io/configuration-snippet: | - more_set_headers "X-Frame-Options: deny"; - more_set_headers "X-XSS-Protection: 1; mode=block"; - nginx.ingress.kubernetes.io/rewrite-target: / -promtail: - enabled: true -grafana: - enabled: false -EOF - -tee /tmp/grafana.yaml << EOF -ingress: - enabled: true - hosts: ["grafana","grafana.jarvis","grafana.jarvis.svc.cluster.local"] - public: true - annotations: - nginx.ingress.kubernetes.io/configuration-snippet: | - more_set_headers "X-Frame-Options: deny"; - more_set_headers "X-XSS-Protection: 1; mode=block"; - nginx.ingress.kubernetes.io/rewrite-target: / - labels: {} - path: / - hosts: - - grafana-jarvis.domain - ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services. - extraPaths: [] - tls: [] -resources: - limits: - cpu: 100m - memory: 128Mi - requests: - cpu: 100m - memory: 128Mi -persistence: - type: pvc - enabled: false - storageClassName: nfs-provisioner - accessModes: - - ReadWriteOnce - size: 10Gi - # annotations: {} - finalizers: - - kubernetes.io/pvc-protection -adminUser: admin -# adminPassword: strongpassword -admin: - existingSecret: "" - userKey: admin-user - passwordKey: admin-password -datasources: - datasources.yaml: - apiVersion: 1 - datasources: - - name: Loki - type: loki - access: proxy - url: http://loki.loki-stack:3100 - version: 1 -EOF +./tools/deployment/common/wait-for-pods.sh ${TEKTON_NS} +./tools/deployment/common/wait-for-pods.sh ${HARBOR_NS} # install loki-stack with Loki and Promtail from Grafana helm charts repo helm upgrade --install loki grafana/loki-stack \ --namespace=${LOKI_NS} \ - --values=/tmp/loki.yaml \ ${EXTRA_HELM_ARGS_LOKI_STACK} -kubectl wait --for=condition=ready pod --timeout=600s --namespace ${LOKI_NS} --all +./tools/deployment/common/wait-for-pods.sh ${LOKI_NS} # install Grafana from Grafana helm charts repo helm upgrade --install grafana grafana/grafana \ --namespace=${GRAFANA_NS} \ - --values=/tmp/grafana.yaml \ ${EXTRA_HELM_ARGS_GRAFANA} -kubectl wait --for=condition=ready pod --timeout=600s --namespace ${GRAFANA_NS} --all +./tools/deployment/common/wait-for-pods.sh ${GRAFANA_NS} diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 2a280010..590055a4 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -26,6 +26,8 @@ description: Runs Tekton charts nodeset: airship-charts-single-node vars: + aio_params: + feature_gates: "tls storageclass nfs harbor readonly loki-stack grafana" collect_kubernetes_logs_namespace: tekton-pipelines gate_scripts: - ./tools/utils/aio-setup.sh