[etcd] Add cronjob with database compaction

etcd database need to be periodically compacted and defrag
This patch adds jobs to perform required maintenance actions
automatically.

Co-Authored-By: Oleh Hryhorov <ohryhorov@mirantis.com>

Change-Id: I31b48bb198f7322c343c7d0171322759893e374f

etcd/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v3.4.3
 description: OpenStack-Helm etcd
 name: etcd
-version: 0.1.8
+version: 0.1.9
 home: https://coreos.com/etcd/
 icon: https://raw.githubusercontent.com/CloudCoreo/etcd-cluster/master/images/icon.png
 sources:

etcd/templates/bin/_etcd-db-compact.sh.tpl

@@ -0,0 +1,47 @@
+#!/bin/sh
+
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -x
+
+export ETCDCTL_API=3
+
+{{- if .Values.jobs.db_compact.command_timeout }}
+COMMAND_TIMEOUT='--command-timeout={{ .Values.jobs.db_compact.command_timeout }}'
+{{- else }}
+COMMAND_TIMEOUT=''
+{{- end }}
+
+ENDPOINTS=$(etcdctl member list --endpoints=http://${ETCD_SERVICE_HOST}:${ETCD_SERVICE_PORT} ${COMMAND_TIMEOUT}| cut -d, -f5 | sed -e 's/ //g' | paste -sd ',')
+
+etcdctl --endpoints=${ENDPOINTS} endpoint status --write-out="table" ${COMMAND_TIMEOUT}
+
+rev=$(etcdctl --endpoints=http://${ETCD_SERVICE_HOST}:${ETCD_SERVICE_PORT} endpoint status --write-out="json" ${COMMAND_TIMEOUT}| egrep -o '"revision":[0-9]*' | egrep -o '[0-9].*')
+compact_result=$(etcdctl compact --physical=true --endpoints=${ENDPOINTS} $rev ${COMMAND_TIMEOUT} 2>&1 > /dev/null)
+compact_res=$?
+
+if [[ $compact_res -ne 0 ]]; then
+    match_pattern=$(echo ${compact_result} | egrep '(mvcc: required revision has been compacted.*$)')
+    match_pattern_res=$?
+    if [[ $match_pattern_res -eq 0 ]]; then
+        exit 0
+    else
+        echo "Failed to compact database: $compact_result"
+        exit $compact_res
+    fi
+else
+    etcdctl defrag --endpoints=${ENDPOINTS} ${COMMAND_TIMEOUT}
+    etcdctl --endpoints=${ENDPOINTS} endpoint status --write-out="table" ${COMMAND_TIMEOUT}
+fi

etcd/templates/configmap-bin.yaml

@@ -27,6 +27,10 @@ data:
 {{- end }}
   etcd.sh: |
 {{ tuple "bin/_etcd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- if .Values.manifests.cron_job_db_compact }}
+  etcd-db-compact.sh: |
+{{ tuple "bin/_etcd-db-compact.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
   etcd-healthcheck.sh: |
 {{ tuple "bin/_etcd-healthcheck.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}

etcd/templates/cron-job-db-compact.yaml

@@ -0,0 +1,75 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.cron_job_db_compact }}
+{{- $envAll := . }}
+
+{{- $configMapBinName := printf "%s-%s" $envAll.Release.Name "etcd-bin"  }}
+
+{{- $serviceAccountName := "etcd-db-compact" }}
+{{ tuple $envAll "db_compact" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: etcd-db-compaction
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+spec:
+  schedule: {{ .Values.jobs.db_compact.cron | quote }}
+  successfulJobsHistoryLimit: {{ .Values.jobs.db_compact.history.success }}
+  failedJobsHistoryLimit: {{ .Values.jobs.db_compact.history.failed }}
+  {{- if .Values.jobs.db_compact.starting_deadline }}
+  startingDeadlineSeconds: {{ .Values.jobs.db_compact.starting_deadline }}
+  {{- end }}
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    metadata:
+      labels:
+{{ tuple $envAll "etcd" "db-compact" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      template:
+        metadata:
+          labels:
+{{ tuple $envAll "etcd" "db-compact" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }}
+        spec:
+{{ dict "envAll" $envAll "application" "etcd_db_compact" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }}
+          serviceAccountName: {{ $serviceAccountName }}
+          restartPolicy: OnFailure
+          nodeSelector:
+            {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+          initContainers:
+{{ tuple $envAll "db_compact" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }}
+          containers:
+            - name: etcd-db-compact
+{{ tuple $envAll "etcd_db_compact" | include "helm-toolkit.snippets.image" | indent 14 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.db_compact | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
+{{ dict "envAll" $envAll "application" "etcd_db_compact" "container" "etcd_db_compact" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
+              command:
+                - /tmp/etcd-db-compact.sh
+              volumeMounts:
+                - name: pod-tmp
+                  mountPath: /tmp
+                - name: etcd-bin
+                  mountPath: /tmp/etcd-db-compact.sh
+                  subPath: etcd-db-compact.sh
+                  readOnly: true
+          volumes:
+            - name: pod-tmp
+              emptyDir: {}
+            - name: etcd-bin
+              configMap:
+                name: {{ $configMapBinName | quote }}
+                defaultMode: 0555
+{{- end }}

etcd/values.yaml

@@ -21,6 +21,7 @@ images:
     etcd: 'registry.k8s.io/etcd-amd64:3.4.3'
     dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal
     image_repo_sync: docker.io/library/docker:17.07.0
+    etcd_db_compact: 'registry.k8s.io/etcd-amd64:3.4.3'
   pull_policy: "IfNotPresent"
   local_registry:
     active: false
@@ -32,6 +33,9 @@ labels:
   server:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
+  job:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
 
 dependencies:
   dynamic:
@@ -49,6 +53,10 @@ dependencies:
           service: local_image_registry
     etcd:
       jobs: null
+    db_compact:
+      services:
+        - endpoint: internal
+          service: etcd
 
 pod:
   env:
@@ -63,6 +71,17 @@ pod:
         etcd:
           runAsUser: 0
           readOnlyRootFilesystem: false
+    etcd_db_compact:
+      pod:
+        runAsUser: 65534
+        runAsNonRoot: true
+        allowPrivilegeEscalation: false
+      container:
+        etcd_db_compact:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+              - ALL
   mandatory_access_control:
     type: apparmor
     etcd:
@@ -110,6 +129,10 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+      db_compact:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
 
 secrets:
   oci_image_registry:
@@ -177,6 +200,17 @@ volume:
   class_name: general
   size: 5Gi
 
+jobs:
+  db_compact:
+    cron: "1 */2 * * *"
+    starting_deadline: 600
+    # Timeout have to be set the same format
+    # as it is for etcdctl 120s, 1m etc.
+    command_timeout: 120s
+    history:
+      success: 3
+      failed: 1
+
 manifests:
   configmap_bin: true
   statefulset: true
@@ -184,4 +218,5 @@ manifests:
   secret_registry: true
   service: true
   service_discovery: true
+  cron_job_db_compact: false
 ...

releasenotes/notes/etcd.yaml

@@ -9,4 +9,5 @@ etcd:
   - 0.1.6 Update kubernetes registry to registry.k8s.io
   - 0.1.7 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default
   - 0.1.8 Switch etcd to staetefulset
+  - 0.1.9 Adding cronjob with etcd compaction
 ...