From 5b04ac3aae91851dcefbf87c1edf61c56c1cb3f7 Mon Sep 17 00:00:00 2001 From: Vasyl Saienko Date: Sat, 14 Sep 2024 10:32:53 +0000 Subject: [PATCH] [etcd] Add cronjob with database compaction etcd database need to be periodically compacted and defrag This patch adds jobs to perform required maintenance actions automatically. Co-Authored-By: Oleh Hryhorov Change-Id: I31b48bb198f7322c343c7d0171322759893e374f --- etcd/Chart.yaml | 2 +- etcd/templates/bin/_etcd-db-compact.sh.tpl | 47 ++++++++++++++ etcd/templates/configmap-bin.yaml | 4 ++ etcd/templates/cron-job-db-compact.yaml | 75 ++++++++++++++++++++++ etcd/values.yaml | 35 ++++++++++ releasenotes/notes/etcd.yaml | 1 + 6 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 etcd/templates/bin/_etcd-db-compact.sh.tpl create mode 100644 etcd/templates/cron-job-db-compact.yaml diff --git a/etcd/Chart.yaml b/etcd/Chart.yaml index 7c7f7a871..cc658e205 100644 --- a/etcd/Chart.yaml +++ b/etcd/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v3.4.3 description: OpenStack-Helm etcd name: etcd -version: 0.1.8 +version: 0.1.9 home: https://coreos.com/etcd/ icon: https://raw.githubusercontent.com/CloudCoreo/etcd-cluster/master/images/icon.png sources: diff --git a/etcd/templates/bin/_etcd-db-compact.sh.tpl b/etcd/templates/bin/_etcd-db-compact.sh.tpl new file mode 100644 index 000000000..ff6af04f4 --- /dev/null +++ b/etcd/templates/bin/_etcd-db-compact.sh.tpl @@ -0,0 +1,47 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -x + +export ETCDCTL_API=3 + +{{- if .Values.jobs.db_compact.command_timeout }} +COMMAND_TIMEOUT='--command-timeout={{ .Values.jobs.db_compact.command_timeout }}' +{{- else }} +COMMAND_TIMEOUT='' +{{- end }} + +ENDPOINTS=$(etcdctl member list --endpoints=http://${ETCD_SERVICE_HOST}:${ETCD_SERVICE_PORT} ${COMMAND_TIMEOUT}| cut -d, -f5 | sed -e 's/ //g' | paste -sd ',') + +etcdctl --endpoints=${ENDPOINTS} endpoint status --write-out="table" ${COMMAND_TIMEOUT} + +rev=$(etcdctl --endpoints=http://${ETCD_SERVICE_HOST}:${ETCD_SERVICE_PORT} endpoint status --write-out="json" ${COMMAND_TIMEOUT}| egrep -o '"revision":[0-9]*' | egrep -o '[0-9].*') +compact_result=$(etcdctl compact --physical=true --endpoints=${ENDPOINTS} $rev ${COMMAND_TIMEOUT} 2>&1 > /dev/null) +compact_res=$? + +if [[ $compact_res -ne 0 ]]; then + match_pattern=$(echo ${compact_result} | egrep '(mvcc: required revision has been compacted.*$)') + match_pattern_res=$? + if [[ $match_pattern_res -eq 0 ]]; then + exit 0 + else + echo "Failed to compact database: $compact_result" + exit $compact_res + fi +else + etcdctl defrag --endpoints=${ENDPOINTS} ${COMMAND_TIMEOUT} + etcdctl --endpoints=${ENDPOINTS} endpoint status --write-out="table" ${COMMAND_TIMEOUT} +fi diff --git a/etcd/templates/configmap-bin.yaml b/etcd/templates/configmap-bin.yaml index 905d34619..d5407333b 100644 --- a/etcd/templates/configmap-bin.yaml +++ b/etcd/templates/configmap-bin.yaml @@ -27,6 +27,10 @@ data: {{- end }} etcd.sh: | {{ tuple "bin/_etcd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- if .Values.manifests.cron_job_db_compact }} + etcd-db-compact.sh: | +{{ tuple "bin/_etcd-db-compact.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} etcd-healthcheck.sh: | {{ tuple "bin/_etcd-healthcheck.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/etcd/templates/cron-job-db-compact.yaml b/etcd/templates/cron-job-db-compact.yaml new file mode 100644 index 000000000..80a64e11c --- /dev/null +++ b/etcd/templates/cron-job-db-compact.yaml @@ -0,0 +1,75 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cron_job_db_compact }} +{{- $envAll := . }} + +{{- $configMapBinName := printf "%s-%s" $envAll.Release.Name "etcd-bin" }} + +{{- $serviceAccountName := "etcd-db-compact" }} +{{ tuple $envAll "db_compact" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: etcd-db-compaction + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + schedule: {{ .Values.jobs.db_compact.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.db_compact.history.success }} + failedJobsHistoryLimit: {{ .Values.jobs.db_compact.history.failed }} + {{- if .Values.jobs.db_compact.starting_deadline }} + startingDeadlineSeconds: {{ .Values.jobs.db_compact.starting_deadline }} + {{- end }} + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: +{{ tuple $envAll "etcd" "db-compact" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + template: + metadata: + labels: +{{ tuple $envAll "etcd" "db-compact" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: +{{ dict "envAll" $envAll "application" "etcd_db_compact" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "db_compact" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + containers: + - name: etcd-db-compact +{{ tuple $envAll "etcd_db_compact" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.db_compact | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "etcd_db_compact" "container" "etcd_db_compact" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + command: + - /tmp/etcd-db-compact.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: etcd-bin + mountPath: /tmp/etcd-db-compact.sh + subPath: etcd-db-compact.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: etcd-bin + configMap: + name: {{ $configMapBinName | quote }} + defaultMode: 0555 +{{- end }} diff --git a/etcd/values.yaml b/etcd/values.yaml index fdfe8ffee..effaa7a6d 100644 --- a/etcd/values.yaml +++ b/etcd/values.yaml @@ -21,6 +21,7 @@ images: etcd: 'registry.k8s.io/etcd-amd64:3.4.3' dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal image_repo_sync: docker.io/library/docker:17.07.0 + etcd_db_compact: 'registry.k8s.io/etcd-amd64:3.4.3' pull_policy: "IfNotPresent" local_registry: active: false @@ -32,6 +33,9 @@ labels: server: node_selector_key: openstack-control-plane node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled dependencies: dynamic: @@ -49,6 +53,10 @@ dependencies: service: local_image_registry etcd: jobs: null + db_compact: + services: + - endpoint: internal + service: etcd pod: env: @@ -63,6 +71,17 @@ pod: etcd: runAsUser: 0 readOnlyRootFilesystem: false + etcd_db_compact: + pod: + runAsUser: 65534 + runAsNonRoot: true + allowPrivilegeEscalation: false + container: + etcd_db_compact: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL mandatory_access_control: type: apparmor etcd: @@ -110,6 +129,10 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + db_compact: + requests: + memory: "128Mi" + cpu: "100m" secrets: oci_image_registry: @@ -177,6 +200,17 @@ volume: class_name: general size: 5Gi +jobs: + db_compact: + cron: "1 */2 * * *" + starting_deadline: 600 + # Timeout have to be set the same format + # as it is for etcdctl 120s, 1m etc. + command_timeout: 120s + history: + success: 3 + failed: 1 + manifests: configmap_bin: true statefulset: true @@ -184,4 +218,5 @@ manifests: secret_registry: true service: true service_discovery: true + cron_job_db_compact: false ... diff --git a/releasenotes/notes/etcd.yaml b/releasenotes/notes/etcd.yaml index 209a6d4cd..2d1c09a04 100644 --- a/releasenotes/notes/etcd.yaml +++ b/releasenotes/notes/etcd.yaml @@ -9,4 +9,5 @@ etcd: - 0.1.6 Update kubernetes registry to registry.k8s.io - 0.1.7 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default - 0.1.8 Switch etcd to staetefulset + - 0.1.9 Adding cronjob with etcd compaction ...