From f94aed3c7a0cbef1b3ad3362a511ac7ab56315cc Mon Sep 17 00:00:00 2001
From: "Gupta, Sangeet (sg774j)" <sg774j@att.com>
Date: Mon, 26 Apr 2021 18:18:41 +0000
Subject: [PATCH] cert-rotation: New chart for certificate rotation

This chart creates a cronjob which monitors the expiry of the
certificates created by jetstack cert-manager. It rotates the
certificates and restarts the pods that mounts the certificate
secrets so that the new certificate can take effect.

Change-Id: I492b5f319cf0f2e7ccbbcf516953e17aafc1c59f
---
 cert-rotation/Chart.yaml                      |  20 ++
 cert-rotation/requirements.yaml               |  18 ++
 .../templates/bin/_rotate-certs.sh.tpl        | 207 ++++++++++++++++++
 cert-rotation/templates/configmap-bin.yaml    |  25 +++
 .../templates/cron-job-cert-rotate.yaml       | 120 ++++++++++
 cert-rotation/templates/job-cert-rotate.yaml  | 107 +++++++++
 cert-rotation/values.yaml                     |  61 ++++++
 releasenotes/notes/cert-rotation.yaml         |   4 +
 8 files changed, 562 insertions(+)
 create mode 100644 cert-rotation/Chart.yaml
 create mode 100644 cert-rotation/requirements.yaml
 create mode 100644 cert-rotation/templates/bin/_rotate-certs.sh.tpl
 create mode 100644 cert-rotation/templates/configmap-bin.yaml
 create mode 100644 cert-rotation/templates/cron-job-cert-rotate.yaml
 create mode 100644 cert-rotation/templates/job-cert-rotate.yaml
 create mode 100644 cert-rotation/values.yaml
 create mode 100644 releasenotes/notes/cert-rotation.yaml

diff --git a/cert-rotation/Chart.yaml b/cert-rotation/Chart.yaml
new file mode 100644
index 000000000..2b62e1481
--- /dev/null
+++ b/cert-rotation/Chart.yaml
@@ -0,0 +1,20 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+apiVersion: v1
+appVersion: "1.0"
+description: Rotate the certificates generated by cert-manager
+home: https://cert-manager.io/
+name: cert-rotation
+version: 0.1.0
+...
diff --git a/cert-rotation/requirements.yaml b/cert-rotation/requirements.yaml
new file mode 100644
index 000000000..19b0d6992
--- /dev/null
+++ b/cert-rotation/requirements.yaml
@@ -0,0 +1,18 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+dependencies:
+  - name: helm-toolkit
+    repository: http://localhost:8879/charts
+    version: ">= 0.1.0"
+...
diff --git a/cert-rotation/templates/bin/_rotate-certs.sh.tpl b/cert-rotation/templates/bin/_rotate-certs.sh.tpl
new file mode 100644
index 000000000..48683e421
--- /dev/null
+++ b/cert-rotation/templates/bin/_rotate-certs.sh.tpl
@@ -0,0 +1,207 @@
+#!/bin/bash
+
+set -e
+
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+
+COMMAND="${@:-rotate_job}"
+
+namespace={{ .Release.Namespace }}
+minDaysToExpiry={{ .Values.jobs.rotate.max_days_to_expiry }}
+
+rotateBefore=$(($(date +%s) + (86400*$minDaysToExpiry)))
+
+# Return Code, initialized to success
+rc=0
+
+function rotate_and_get_certs_list(){
+    # Rotate the certificates if the expiry date of certificates is within the
+    # max_days_to_expiry days
+
+    # List of secret and certificates rotated
+    local -n secRotated=$1
+    deleteAllSecrets=$2
+    certRotated=()
+
+    for certificate in $(kubectl get certificates -n ${namespace} --no-headers | awk '{ print $1 }')
+    do
+        certInfo=($(kubectl get certificate -n ${namespace} ${certificate} -o json | jq -r '.spec["secretName"],.status["notAfter"]'))
+        secretName=${certInfo[0]}
+        notAfter=$(date -d"${certInfo[1]}" '+%s')
+        deleteSecret=false
+        if ${deleteAllSecrets} || [ ${rotateBefore} -gt ${notAfter} ]
+        then
+            # Rotate the certificates/secrets and add to list.
+            echo "Deleting secret: ${secretName}"
+            kubectl delete secret -n ${namespace} $secretName
+            secRotated+=(${secretName})
+            certRotated+=(${certificate})
+        fi
+    done
+
+    # Ensure certificates are re-issued
+    if [ ! -z ${certRotated} ]
+    then
+        for cert in ${certRotated[@]}
+        do
+            counter=0
+            while [ "$(kubectl get certificate -n ${namespace} ${cert} -o json | jq -r '.status.conditions[].status')" != "True" ]
+            do
+                # Wait for secret to become ready. Wait for 300 seconds maximum. Sleep for 10 seconds
+                if [ ${counter} -ge 30 ]
+                then
+                    echo "ERROR: Rotated certificate  ${cert} in ${namespace} is not ready."
+                    # Set return code to error and continue so that the certificates that are
+                    # rotated successfully are deployed.
+                    rc=1
+                    break
+                fi
+                echo "Rotated certificate ${cert} in ${namespace} is not ready yet ... waiting"
+                counter+=(${counter+=1})
+                sleep 10
+            done
+
+        done
+    fi
+}
+
+function get_cert_list_rotated_by_cert_manager_rotate(){
+
+    local -n secRotated=$1
+
+    # Get the time when the last cron job was run successfully
+    lastCronTime=$(kubectl get jobs -n ${namespace} --no-headers -l application=cert-manager,component=cert-rotate -o json | jq -r '.items[] | select(.status.succeeded != null) | .status.completionTime' | sort -r | head -n 1)
+
+    if [ ! -z ${lastCronTime} ]
+    then
+        lastCronTimeSec=$(date -d"${lastCronTime}" '+%s')
+
+        for certificate in $(kubectl get certificates -n ${namespace} --no-headers | awk '{ print $1 }')
+        do
+            certInfo=($(kubectl get certificate -n ${namespace} ${certificate} -o json | jq -r '.spec["secretName"],.status["notBefore"]'))
+            secretName=${certInfo[0]}
+            notBefore=$(date -d"${certInfo[1]}" '+%s')
+
+            # if the certificate was created after last cronjob run means it was
+            # rotated by the cert-manager, add to the list.
+            if [[ ${notBefore} -gt ${lastCronTimeSec} ]]
+            then
+                secRotated+=(${secretName})
+            fi
+        done
+    fi
+}
+
+function restart_the_pods(){
+
+    local -n secRotated=$1
+
+    if [ -z ${secRotated} ]
+    then
+        echo "All certificates are still valid in ${namespace} namespace. No pod needs restart"
+        exit 0
+    fi
+
+    # Restart the pods using kubernetes rollout restart. This will restarts the applications
+    # with zero downtime.
+    for kind in statefulset deployment daemonset
+    do
+        # Need to find which kinds mounts the secret that has been rotated. To do this
+        # for a kind (statefulset, deployment, or daemonset)
+        # - get the name of the kind (which will index 1 = idx=0 of the output)
+        # - get the names of the secrets mounted on this kind (which will be index 2 = idx+1)
+        # - find if tls.crt was mounted to the container: get the subpaths of volumeMount in
+        #   the container and grep for tls.crt. (This will be index 2 = idx+2)
+
+        resource=($(kubectl get ${kind} -n ${namespace} -o custom-columns='NAME:.metadata.name,SECRETS:.spec.template.spec.volumes[*].secret.secretName,TLS:.spec.template.spec.containers[*].volumeMounts[*].subPath' --no-headers | grep tls.crt))
+
+        idx=0
+        while [[ $idx -lt ${#resource[@]} ]]
+        do
+            # Name of the kind
+            resourceName=${resource[$idx]}
+
+            # List of secrets mounted to this kind
+            resourceSecrets=${resource[$idx+1]}
+
+            # For each secret mounted to this kind, check if it was rotated (present in
+            # the list secRotated) and if it was, then trigger rolling restart for this kind.
+            for secret in ${resourceSecrets//,/ }
+            do
+                if [[ "${secRotated[@]}" =~ "${secret}" ]]
+                then
+                    echo "Restarting ${kind} ${resourceName} in ${namespace} namespace."
+                    kubectl rollout restart -n ${namespace} ${kind} ${resourceName}
+                    break
+                fi
+            done
+
+            # Since we have 3 custom colums in the output, every 4th index will be start of new tuple.
+            # Jump to the next tuple.
+            idx=$((idx+3))
+        done
+    done
+}
+
+function rotate_cron(){
+    # Rotate cronjob invoked this script.
+    # 1. If the expiry date of certificates is within the max_days_to_expiry days
+    #    the rotate the certificates and restart the pods
+    # 2. Else if the certificates were rotated by cert-manager, then restart
+    #    the pods.
+
+    secretsRotated=()
+    deleteAllSecrets=false
+
+    rotate_and_get_certs_list secretsRotated $deleteAllSecrets
+
+    if [ ! -z ${secretsRotated} ]
+    then
+        # Certs rotated, restart pods
+        restart_the_pods secretsRotated
+    else
+        # Check if the certificates were rotated by the cert-manager and get the list of
+        # rotated certificates so that the corresponding pods can be restarted
+        get_cert_list_rotated_by_cert_manager_rotate secretsRotated
+        if [ ! -z ${secretsRotated} ]
+        then
+            restart_the_pods secretsRotated
+        else
+            echo "All certificates are still valid in ${namespace} namespace"
+        fi
+    fi
+}
+
+function rotate_job(){
+    # Rotate job invoked this script.
+    # 1. Rotate all certificates by deleting the secrets and restart the pods
+
+    secretsRotated=()
+    deleteAllSecrets=true
+
+    rotate_and_get_certs_list secretsRotated $deleteAllSecrets
+
+    if [ ! -z ${secretsRotated} ]
+    then
+        # Certs rotated, restart pods
+        restart_the_pods secretsRotated
+    else
+        echo "All certificates are still valid in ${namespace} namespace"
+    fi
+}
+
+$COMMAND
+exit ${rc}
diff --git a/cert-rotation/templates/configmap-bin.yaml b/cert-rotation/templates/configmap-bin.yaml
new file mode 100644
index 000000000..e13463a6a
--- /dev/null
+++ b/cert-rotation/templates/configmap-bin.yaml
@@ -0,0 +1,25 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_bin }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cert-rotate-bin
+data:
+  rotate-certs.sh: |
+{{ tuple "bin/_rotate-certs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ end }}
diff --git a/cert-rotation/templates/cron-job-cert-rotate.yaml b/cert-rotation/templates/cron-job-cert-rotate.yaml
new file mode 100644
index 000000000..46a2e2366
--- /dev/null
+++ b/cert-rotation/templates/cron-job-cert-rotate.yaml
@@ -0,0 +1,120 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.cron_job_cert_rotate}}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := "cert-rotate-cron" }}
+{{ tuple $envAll "cert_rotate" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+    - cert-manager.io
+    resources:
+      - certificates
+    verbs:
+      - get
+      - list
+      - update
+      - patch
+  - apiGroups:
+    - "*"
+    resources:
+      - pods
+      - secrets
+      - jobs
+      - statefulsets
+      - daemonsets
+      - deployments
+    verbs:
+      - get
+      - list
+      - update
+      - patch
+      - delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: cert-rotate
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+  labels:
+{{ tuple $envAll "cert-manager" "cert-rotate-cron" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  suspend: {{ .Values.jobs.rotate.suspend }}
+  schedule: {{ .Values.jobs.rotate.cron | quote }}
+  successfulJobsHistoryLimit: {{ .Values.jobs.rotate.history.success }}
+  failedJobsHistoryLimit: {{ .Values.jobs.rotate.history.failed }}
+{{- if .Values.jobs.rotate.starting_deadline }}
+  startingDeadlineSeconds: {{ .Values.jobs.rotate.starting_deadline }}
+{{- end }}
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    metadata:
+      labels:
+{{ tuple $envAll "cert-manager" "cert-rotate" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      template:
+        metadata:
+          labels:
+{{ tuple $envAll "cert-manager" "cert-rotate" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
+        spec:
+          serviceAccountName: {{ $serviceAccountName }}
+{{ dict "envAll" $envAll "application" "cert_rotate" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }}
+          restartPolicy: OnFailure
+          nodeSelector:
+            {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+          initContainers:
+{{ tuple $envAll "cert_rotate" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }}
+          containers:
+            - name: cert-rotate
+{{ tuple $envAll "cert_rotation" | include "helm-toolkit.snippets.image" | indent 14 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.cert_rotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
+{{ dict "envAll" $envAll "application" "cert_rotate" "container" "cert_rotate" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
+              command:
+                - /tmp/rotate-certs.sh
+                - rotate_cron
+              volumeMounts:
+                - name: pod-tmp
+                  mountPath: /tmp
+                - name: cert-rotate-bin
+                  mountPath: /tmp/rotate-certs.sh
+                  subPath: rotate-certs.sh
+                  readOnly: true
+          volumes:
+            - name: pod-tmp
+              emptyDir: {}
+            - name: cert-rotate-bin
+              configMap:
+                name: cert-rotate-bin
+                defaultMode: 0555
+{{- end }}
diff --git a/cert-rotation/templates/job-cert-rotate.yaml b/cert-rotation/templates/job-cert-rotate.yaml
new file mode 100644
index 000000000..f508a7d9d
--- /dev/null
+++ b/cert-rotation/templates/job-cert-rotate.yaml
@@ -0,0 +1,107 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_cert_rotate}}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := "cert-rotate-job" }}
+{{ tuple $envAll "cert_rotate" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+    - cert-manager.io
+    resources:
+      - certificates
+    verbs:
+      - get
+      - list
+      - update
+      - patch
+  - apiGroups:
+    - "*"
+    resources:
+      - pods
+      - secrets
+      - jobs
+      - statefulsets
+      - daemonsets
+      - deployments
+    verbs:
+      - get
+      - list
+      - update
+      - patch
+      - delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cert-rotate-job
+  labels:
+{{ tuple $envAll "cert-manager" "cert-rotate-job" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "cert-manager" "cert-rotate" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+      annotations:
+{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+{{ dict "envAll" $envAll "application" "cert_rotate" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+      initContainers:
+{{ tuple $envAll "cert_rotate" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }}
+      containers:
+        - name: cert-rotate
+{{ tuple $envAll "cert_rotation" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.cert_rotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "cert_rotate" "container" "cert_rotate" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - /tmp/rotate-certs.sh
+            - rotate_job
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: cert-rotate-bin
+              mountPath: /tmp/rotate-certs.sh
+              subPath: rotate-certs.sh
+              readOnly: true
+      volumes:
+        - name: pod-tmp
+          emptyDir: {}
+        - name: cert-rotate-bin
+          configMap:
+            name: cert-rotate-bin
+            defaultMode: 0555
+{{- end }}
diff --git a/cert-rotation/values.yaml b/cert-rotation/values.yaml
new file mode 100644
index 000000000..dc9a59208
--- /dev/null
+++ b/cert-rotation/values.yaml
@@ -0,0 +1,61 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+images:
+  tags:
+    cert_rotation: 'docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic'
+    dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
+  local_registry:
+    active: false
+labels:
+  job:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+jobs:
+  rotate:
+    # Run at 1:00AM on 1st of each month
+    cron: "0 1 1 * *"
+    starting_deadline: 600
+    history:
+      success: 3
+      failed: 1
+    # Number of day before expiry should certs be rotated.
+    max_days_to_expiry: 45
+    suspend: false
+pod:
+  security_context:
+    cert_rotate:
+      pod:
+        runAsUser: 42424
+      container:
+        cert_rotate:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+  resources:
+    enabled: false
+    jobs:
+      cert_rotate:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+dependencies:
+  static:
+    cert_rotate: null
+manifests:
+  configmap_bin: true
+  cron_job_cert_rotate: false
+  job_cert_rotate: false
+...
diff --git a/releasenotes/notes/cert-rotation.yaml b/releasenotes/notes/cert-rotation.yaml
new file mode 100644
index 000000000..93cb4381a
--- /dev/null
+++ b/releasenotes/notes/cert-rotation.yaml
@@ -0,0 +1,4 @@
+---
+cert-rotation:
+  - 0.1.0 Initial Chart
+...