From 52151ef9fde824747c65ac73c881f49624fc7b51 Mon Sep 17 00:00:00 2001
From: Sirisha Gopigiri <sirishagopigiri@gmail.com>
Date: Fri, 4 Sep 2020 12:40:13 +0530
Subject: [PATCH] Zuul gate to check rotate-sa-token and certificate expiry
 commands

This Patch Set uses docker zuul gate and
checks the certificate expiration
and certiticate rotate-sa-token commands

Change-Id: I76f902d4fcacdbfe168abd58ec707282e46a3f91
---
 .../provider_common/03_install_pip.sh         |  15 ++
 .../provider_common/04_install_yq.sh          |  15 ++
 .../41_check_certificate_expiration.sh        | 133 ++++++++++++++++++
 .../provider_common/42_rotate_sa_token.sh     |  67 +++++++++
 zuul.d/jobs.yaml                              |   4 +
 5 files changed, 234 insertions(+)
 create mode 100755 tools/deployment/provider_common/03_install_pip.sh
 create mode 100755 tools/deployment/provider_common/04_install_yq.sh
 create mode 100755 tools/deployment/provider_common/41_check_certificate_expiration.sh
 create mode 100755 tools/deployment/provider_common/42_rotate_sa_token.sh

diff --git a/tools/deployment/provider_common/03_install_pip.sh b/tools/deployment/provider_common/03_install_pip.sh
new file mode 100755
index 000000000..189ef2e76
--- /dev/null
+++ b/tools/deployment/provider_common/03_install_pip.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -ex
+
+sudo apt-get install python3-pip -y
diff --git a/tools/deployment/provider_common/04_install_yq.sh b/tools/deployment/provider_common/04_install_yq.sh
new file mode 100755
index 000000000..3a613b26e
--- /dev/null
+++ b/tools/deployment/provider_common/04_install_yq.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -ex
+
+sudo pip3 install yq
diff --git a/tools/deployment/provider_common/41_check_certificate_expiration.sh b/tools/deployment/provider_common/41_check_certificate_expiration.sh
new file mode 100755
index 000000000..02d669a62
--- /dev/null
+++ b/tools/deployment/provider_common/41_check_certificate_expiration.sh
@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"}
+export TARGET_CLUSTER_NAME=${TARGET_CLUSTER_NAME:-"target-cluster"}
+export KUBECONFIG_TARGET_CONTEXT=${KUBECONFIG_TARGET_CONTEXT:-"target-cluster"}
+
+check_nodecerts(){
+    nodecerts=""
+    if $2; then
+        nodecerts=$(airshipctl --kubeconfig "/tmp/${TARGET_CLUSTER_NAME}.kubeconfig" cluster check-certificate-expiration --threshold $1 -o json | jq .nodeCerts)
+    else
+        nodecerts=$(airshipctl --kubeconfig ${KUBECONFIG} cluster check-certificate-expiration --threshold $1 -o json | jq .nodeCerts)
+    fi
+    nodecert=$(echo $nodecerts | jq '. | length')
+    if [ -z $nodecert ]; then
+        echo "Unable to verify node certificate expiration. Exiting!"
+        exit 1
+    else
+        verify=false
+        for ((i=0;i<nodecert;i++)); do
+            nodeName=$(echo $nodecerts | jq -r ".[$i].name")
+            if [[ $3 == $nodeName ]]; then
+                echo "Node certificate on $nodeName verified"
+                verify=true
+                break
+            fi
+        done
+        if $verify; then
+            echo "Node certificate annotation verified!"
+        else
+            echo "Node certificate verification failed!"
+	    exit 1
+        fi
+    fi
+}
+
+check_tls_certs(){
+    if [[ $1 = "json" ]]; then
+        tlscerts=$(airshipctl cluster check-certificate-expiration --threshold 365 -o json --kubeconfig ${KUBECONFIG} | jq '.tlsSecrets')
+    else
+	tlscerts=$(airshipctl cluster check-certificate-expiration --threshold 365 -o yaml --kubeconfig ${KUBECONFIG} | yq '.tlsSecrets')
+    fi
+    tlscert=$(echo $tlscerts | jq '. | length')
+    secret_names=$(kubectl --kubeconfig ${KUBECONFIG} get secrets -A --field-selector="type=kubernetes.io/tls" -o jsonpath='{.items[*].metadata.name}')
+
+    if [[ $tlscert -gt 0 ]]; then
+        for ((i=0;i<tlscert;i++)); do
+            secretName=$(echo $tlscerts | jq -r ".[$i].name")
+            if [[ ${secret_names[@]} =~ $secretName ]]; then
+                echo "Secret name matched for 365 day threshold!"
+                continue
+            else
+                echo "Secret name, $secretName not matched for 365 day threshold!"
+                exit 1
+            fi
+        done
+    else
+        echo "No TLS certificate expires in 365 days threshold!"
+        exit 1
+    fi
+}
+
+echo "Checking TLS certificate expiry within 365 days in JSON format"
+check_tls_certs json
+
+echo "Checking TLS certificate expiry within 365 days in YAML format"
+check_tls_certs yaml
+
+echo "Checking kubeconfig certificates"
+kubecerts=$(airshipctl --kubeconfig "/tmp/${TARGET_CLUSTER_NAME}.kubeconfig" cluster check-certificate-expiration --threshold 365 | jq '.kubeconfs')
+kubecert=$(echo $kubecerts | jq '. | length ')
+if [[ $kubecert -gt 0 ]]; then
+    kubecertName=$(echo $kubecerts | jq -r ".[0].secretName")
+    kubecertNamespace=$(echo $kubecerts | jq -r ".[0].secretNamespace")
+    echo "Copying target kubeconfig secret to another namespace to validate if this new kubenconfig will also be listed"
+    kubectl --kubeconfig "/tmp/${TARGET_CLUSTER_NAME}.kubeconfig" create ns testing
+    kubectl --kubeconfig "/tmp/${TARGET_CLUSTER_NAME}.kubeconfig" get secret $kubecertName --namespace $kubecertNamespace --export -o yaml | kubectl --kubeconfig "/tmp/${TARGET_CLUSTER_NAME}.kubeconfig" apply --namespace=testing -f -
+    kubecerts=$(airshipctl --kubeconfig "/tmp/${TARGET_CLUSTER_NAME}.kubeconfig" cluster check-certificate-expiration --threshold 365 -o json | jq .kubeconfs)
+    kubecert=$(echo $kubecerts | jq '. |length ')
+    verify=false
+    for ((i=0;i<kubecert;i++)); do
+        kubecertNewName=$(echo $kubecerts | jq -r ".[$i].secretName")
+	kubecertNewNamespace=$(echo $kubecerts | jq -r ".[$i].secretNamespace")
+	if [[ $kubecertNewNamespace == "testing" ]] && [[ $kubecertNewName == $kubecertName ]]; then
+            echo "Kubecert with name $kubecertName verified in testing namespace !"
+	    verify=true
+	    break
+        fi
+    done
+    kubectl --kubeconfig "/tmp/${TARGET_CLUSTER_NAME}.kubeconfig" delete ns testing
+    if !$verify; then
+        echo "Kubecerts verification failed!"
+        exit 1
+    fi
+else
+    echo "Unable to get kubecertificates!!"
+    exit 1
+fi
+
+
+MgmtNode=$(kubectl --kubeconfig ${KUBECONFIG} get nodes -o jsonpath={.items[0].metadata.name})
+echo "Checking Node Certificate Expiry of $MgmtNode Node"
+
+end=$(date --date='now next Year' '+%b %d, %Y %H:%M %Z' -u)
+kubectl --kubeconfig ${KUBECONFIG} annotate node $MgmtNode cert-expiration="{ admin.conf: $end },{ apiserver: $end },{ apiserver-etcd-client: $end },{ apiserver-kubelet-client: $end },{ controller-manager.conf: $end },{ etcd-healthcheck-client: $end },{ etcd-peer: $end },{ etcd-server: $end },{ front-proxy-client: $end },{ scheduler.conf: $end },{ ca: $end },{ etcd-ca: $end },{ front-proxy-ca: $end}" --overwrite=true
+
+check_nodecerts 365 false $MgmtNode
+
+end=$(date '+%b %d, %Y %H:%M %Z' -u)
+kubectl --kubeconfig ${KUBECONFIG} annotate node $MgmtNode cert-expiration="{ admin.conf: $end },{ apiserver: $end },{ apiserver-etcd-client: $end },{ apiserver-kubelet-client: $end },{ controller-manager.conf: $end },{ etcd-healthcheck-client: $end },{ etcd-peer: $end },{ etcd-server: $end },{ front-proxy-client: $end },{ scheduler.conf: $end },{ ca: $end },{ etcd-ca: $end },{ front-proxy-ca: $end}" --overwrite=true
+
+check_nodecerts 1 false $MgmtNode
+
+TargetNode=$(kubectl get nodes --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT | grep master | awk '{print $1}' | head -1)
+
+end=$(date --date='now next Year' '+%b %d, %Y %H:%M %Z' -u)
+kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT annotate node $TargetNode cert-expiration="{ admin.conf: $end },{ apiserver: $end },{ apiserver-etcd-client: $end },{ apiserver-kubelet-client: $end },{ controller-manager.conf: $end },{ etcd-healthcheck-client: $end },{ etcd-peer: $end },{ etcd-server: $end },{ front-proxy-client: $end },{ scheduler.conf: $end },{ ca: $end },{ etcd-ca: $end },{ front-proxy-ca: $end}" --overwrite=true
+
+check_nodecerts 365 true $TargetNode
diff --git a/tools/deployment/provider_common/42_rotate_sa_token.sh b/tools/deployment/provider_common/42_rotate_sa_token.sh
new file mode 100755
index 000000000..7c5467f9d
--- /dev/null
+++ b/tools/deployment/provider_common/42_rotate_sa_token.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"}
+export KUBECONFIG_TARGET_CONTEXT=${KUBECONFIG_TARGET_CONTEXT:-"target-cluster"}
+
+echo "Getting pod Name in CAPD namespace"
+podName=$(kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT --namespace capi-system get pods -o jsonpath='{.items[0].metadata.name}')
+
+secretName=$(kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get secrets -n capi-system | grep 'kubernetes.io/service-account-token' | grep default |awk '{print $1}' | head -1)
+echo "Checking airshipctl cluster rotate-sa-token in capd-system namespace"
+airshipctl --kubeconfig $KUBECONFIG cluster rotate-sa-token --secret-namespace capi-system --secret-name $secretName
+
+sleep 5
+echo "Checking for ready state of all pods"
+kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT wait -n capi-system --for=condition=Available deploy --all --timeout=600s
+kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get pods -n capi-system
+
+podNameNew=$(kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT --namespace capi-system get pods | grep -i running | awk '{print $1}')
+
+if [[ -z $podNameNew || $podName == $podNameNew ]]; then
+    echo "Rotation of SA token is unsuccessful"
+    exit 1
+fi
+
+echo "Rotate SA token is successful"
+
+echo "Testing in default namespace with Nginx pod"
+kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT create deploy nginx-deployment --image=nginx
+
+kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT wait --for=condition=Available deploy --all --timeout=600s
+
+echo "Getting the current Nginx podname"
+podName=$(kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get pods -o jsonpath='{.items[0].metadata.name}')
+
+secretName=$(kubectl get secret --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT | grep 'kubernetes.io/service-account-token' | grep default |awk '{print $1}' | head -1)
+
+airshipctl cluster rotate-sa-token --kubeconfig $KUBECONFIG --secret-namespace default --secret-name $secretName
+
+sleep 5
+echo "Checking for ready state of all pods"
+kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT wait --for=condition=Available deploy --all --timeout=600s
+kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get pods
+
+podNameNew=$(kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get pods | grep -i running | awk '{print $1}')
+
+if [[ -z $podNameNew || $podName == $podNameNew ]]; then
+    echo "Rotation of SA token is unsuccessful"
+    kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT delete deploy nginx-deployment
+    exit 1
+fi
+echo "Rotate SA token is successful"
+
+kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT delete deploy nginx-deployment
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index a6c959222..59ff8cebc 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -180,6 +180,8 @@
         - ./tools/deployment/01_install_kubectl.sh
         - ./tools/deployment/provider_common/01_install_kind.sh
         - ./tools/deployment/provider_common/02_install_jq.sh
+        - ./tools/deployment/provider_common/03_install_pip.sh
+        - ./tools/deployment/provider_common/04_install_yq.sh
         - CLUSTER=ephemeral-cluster KIND_CONFIG=./tools/deployment/templates/kind-cluster-with-extramounts ./tools/document/start_kind.sh
         - AIRSHIP_CONFIG_METADATA_PATH=manifests/site/docker-test-site/metadata.yaml SITE=docker-test-site EXTERNAL_KUBECONFIG="true" ./tools/deployment/22_test_configs.sh
         - PROVIDER=default SITE=docker-test-site ./tools/deployment/26_deploy_capi_ephemeral_node.sh
@@ -187,6 +189,8 @@
         - KUBECONFIG=/tmp/target-cluster.kubeconfig ./tools/deployment/provider_common/32_cluster_init_target_node.sh
         - ./tools/deployment/provider_common/33_cluster_move_target_node.sh
         - WORKERS_COUNT=2 KUBECONFIG=/tmp/target-cluster.kubeconfig SITE=docker-test-site ./tools/deployment/provider_common/34_deploy_worker_node.sh
+        - KUBECONFIG=/tmp/target-cluster.kubeconfig ./tools/deployment/provider_common/41_check_certificate_expiration.sh
+        - KUBECONFIG=/tmp/target-cluster.kubeconfig ./tools/deployment/provider_common/42_rotate_sa_token.sh
     voting: false
 - job:
     name: airship-airshipctl-publish-image