Add host paths and openstack/swift clients to etcdctl utility

1) Adds etcd on demand job for backup and restore. 2) Adds dbutils implementation to perform manual local and remote backup/restore for etcd. 3) Adds the openstack and swift clients to the etcdctl utlity pod so that this pod can access etcd remote gateway backup files via the swift interface. 4) Adds kubectl to create backup/restore etcd jobs. Change-Id: Iadfaa828366bae3a98552891c24d669f2922e1d1
2020-05-08 18:27:39 +00:00 · 2020-05-08 18:27:39 +00:00 · de0f60253c
commit de0f60253c
parent 1643b82d37
5 changed files with 248 additions and 2 deletions
--- a/charts/etcdctl-utility/templates/bin/utility/_etcd_ondemand_job.sh.tpl
+++ b/charts/etcdctl-utility/templates/bin/utility/_etcd_ondemand_job.sh.tpl
@ -0,0 +1,177 @@
+#!/bin/bash
+
+{{- $envAll := . }}
+
+export ETCD_POD_NAMESPACE=$1
+if [[ $ETCD_POD_NAMESPACE == "" ]]; then
+  echo "No namespace given - cannot spawn ondemand job."
+  exit 1
+fi
+
+export ETCD_RGW_SECRET={{ $envAll.Values.conf.etcd_backup_restore.secrets.kube_system.rgw_secret }}
+export ETCD_CONF_SECRET={{ $envAll.Values.conf.etcd_backup_restore.secrets.kube_system.conf_secret }}
+export ETCD_IMAGE_NAME=$(kubectl get cronjob -n ${ETCD_POD_NAMESPACE} kubernetes-etcd-backup -o yaml -o jsonpath="{range .spec.jobTemplate.spec.template.spec.containers[*]}{.image}{'\n'}{end}" | grep etcdctl-utility)
+export ETCD_BACKUP_BASE_PATH=$(kubectl get secret -o yaml -n ${ETCD_POD_NAMESPACE} ${ETCD_CONF_SECRET} | grep BACKUP_BASE_PATH | awk '{print $2}' | base64 -d)
+
+if [[ $NODE == "" ]];then
+  echo "Cannot find node to run ondemand job from."
+  exit 1
+fi
+
+if [[ $ETCD_IMAGE_NAME == "" ]]; then
+  echo "Cannot find the utility image for populating ETCD_IMAGE_NAME variable."
+  exit 1
+fi
+
+cat <<EOF | kubectl create -n $ETCD_POD_NAMESPACE -f -
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: etcd-ondemand
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+  labels:
+{{ tuple $envAll "etcd-ondemand" "ondemand" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "etcd-ondemand" "ondemand" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      restartPolicy: OnFailure
+      serviceAccountName: kubernetes-etcd-etcd-backup
+      nodeName: ${NODE}
+      containers:
+        - name: etcd-ondemand
+          image: ${ETCD_IMAGE_NAME}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.etcd_ondemand | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /bin/sleep
+            - "1000000"
+          env:
+            - name: ETCDCTL_API
+              value: "{{ .Values.conf.etcd.etcdctl_api }}"
+            - name: ETCDCTL_DIAL_TIMEOUT
+              value: 10s
+            - name: ETCDCTL_CACERT
+              value: /etc/etcd/tls/certs/client-ca.pem
+            - name: ETCDCTL_CERT
+              value: /etc/etcd/tls/certs/anchor-etcd-client.pem
+            - name: ETCDCTL_KEY
+              value: /etc/etcd/tls/keys/anchor-etcd-client-key.pem
+            - name: ETCDCTL_ENDPOINTS
+              value: https://{{ .Values.conf.etcd.endpoints }}:{{ .Values.endpoints.etcd.port.client.default }}
+            - name: ONDEMAND_JOB
+              value: etcd-ondemand
+            - name: ARCHIVE_DIR
+              value: $ETCD_BACKUP_BASE_PATH/db/$ETCD_POD_NAMESPACE/etcd/archive
+            - name: BACKUP_RESTORE_SCOPE
+              value: etcd
+            - name: BACKUP_RESTORE_NAMESPACE_LIST
+              value: $ETCD_POD_NAMESPACE
+            - name: ETCD_BACKUP_BASE_DIR
+              valueFrom:
+                secretKeyRef:
+                  key: BACKUP_BASE_PATH
+                  name: ${ETCD_CONF_SECRET}
+            - name: POD_NAMESPACE
+              value: ${ETCD_POD_NAMESPACE}
+            - name: REMOTE_BACKUP_ENABLED
+              valueFrom:
+                secretKeyRef:
+                  key: REMOTE_BACKUP_ENABLED
+                  name: ${ETCD_CONF_SECRET}
+            - name: CONTAINER_NAME
+              valueFrom:
+                secretKeyRef:
+                  key: REMOTE_BACKUP_CONTAINER
+                  name: ${ETCD_CONF_SECRET}
+            - name: OS_IDENTITY_API_VERSION
+              value: "3"
+            - name: OS_AUTH_URL
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_AUTH_URL
+            - name: OS_REGION_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_REGION_NAME
+            - name: OS_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_USERNAME
+            - name: OS_PROJECT_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_PROJECT_NAME
+            - name: OS_USER_DOMAIN_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_USER_DOMAIN_NAME
+            - name: OS_PROJECT_DOMAIN_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_PROJECT_DOMAIN_NAME
+            - name: OS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_PASSWORD
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: kubernetes-etcd-certs
+              mountPath: /etc/etcd/tls/certs
+            - name: kubernetes-etcd-keys
+              mountPath: /etc/etcd/tls/keys
+            - mountPath: /tmp/restore_etcd.sh
+              name: kubernetes-etcd-bin
+              readOnly: true
+              subPath: restore_etcd.sh
+            - mountPath: /tmp/restore_main.sh
+              name: kubernetes-etcd-bin
+              readOnly: true
+              subPath: restore_main.sh
+            - mountPath: /tmp/backup_etcd.sh
+              name: kubernetes-etcd-bin
+              readOnly: true
+              subPath: backup_etcd.sh
+            - mountPath: /tmp/bin/backup_main.sh
+              name: kubernetes-etcd-bin
+              readOnly: true
+              subPath: backup_main.sh
+            - mountPath: {{ .Values.conf.backup.host_backup_path }}
+              name: kubernetes-etcd-backup-dir
+              subPath: .
+            - name: host-etcd
+              mountPath: /var/lib/etcd
+              subPath: .
+      volumes:
+        - name: pod-tmp
+          emptyDir: {}
+        - name: kubernetes-etcd-certs
+          configMap:
+            name: kubernetes-etcd-certs
+            defaultMode: 0444
+        - name: kubernetes-etcd-keys
+          secret:
+            secretName: kubernetes-etcd-keys
+            defaultMode: 0444
+        - name: kubernetes-etcd-bin
+          configMap:
+            name: kubernetes-etcd-bin
+            defaultMode: 0555
+        - name: kubernetes-etcd-backup-dir
+          hostPath:
+            path: {{ .Values.conf.backup.host_backup_path }}
+        - name: host-etcd
+          hostPath:
+            path: /var/lib/etcd
+EOF
--- a/charts/etcdctl-utility/templates/configmap-bin.yaml
+++ b/charts/etcdctl-utility/templates/configmap-bin.yaml
@ -39,6 +39,12 @@ data:
  utilscli: |
 {{ tuple "bin/utility/_utilscli.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}

+  etcd-ondemand-job.sh: |
+{{ tuple "bin/utility/_etcd_ondemand_job.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
+  dbutils: |
+{{ tuple "bin/utility/_dbutils.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
 {{- if .Values.bootstrap.enabled }}
  bootstrap.sh: |
 {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
--- a/charts/etcdctl-utility/templates/deployment-etcdctl-utility.yaml
+++ b/charts/etcdctl-utility/templates/deployment-etcdctl-utility.yaml
@ -27,6 +27,7 @@ rules:
      - namespaces
      - nodes
      - pods
+      - pods/exec
      - persistentvolumeclaims
      - persistentvolumes
    verbs:
@ -34,6 +35,33 @@ rules:
      - list
      - watch
      - create
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "batch"
+    resources:
+      - cronjobs
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "batch"
+    resources:
+      - jobs
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
@ -96,6 +124,10 @@ spec:
              value: /etc/kubernetes/apiserver/pki/etcd-client-key.pem
            - name: ETCDCTL_ENDPOINTS
              value: https://{{ .Values.conf.etcd.endpoints }}:{{ .Values.endpoints.etcd.port.client.default }}
+            - name: BACKUP_RESTORE_NAMESPACE_LIST
+              value: {{ .Values.conf.etcd_backup_restore.enabled_namespaces | quote }}
+            - name: BACKUP_RESTORE_SCOPE
+              value: "etcd"
          command:
            - /tmp/bootstrap.sh
          volumeMounts:
@ -127,6 +159,14 @@ spec:
              mountPath: /usr/local/bin/etcdctl-utility-rootwrap
              subPath: etcdctl-utility-rootwrap
              readOnly: true
+            - name: etcdctl-utility-bin-utilscli
+              mountPath: /usr/local/bin/dbutils
+              subPath: dbutils
+              readOnly: true
+            - name: etcdctl-utility-bin-utilscli
+              mountPath: /tmp/etcd-ondemand-job.sh
+              subPath: etcd-ondemand-job.sh
+              readOnly: true
            - name: etcdctl-utility-sudoers
              mountPath: /etc/sudoers.d/utilscli-sudo
              subPath: utilscli-sudo
--- a/charts/etcdctl-utility/values.yaml
+++ b/charts/etcdctl-utility/values.yaml
@ -83,6 +83,13 @@ pod:
        limits:
          memory: "1024Mi"
          cpu: "2000m"
+      etcd_ondemand:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
    test:
      etcdctl:
        requests:
@ -92,11 +99,21 @@ pod:
          memory: "1024Mi"
          cpu: "2000m"
 conf:
+  backup:
+    host_backup_path: /var/backups
+  etcd_backup_restore:
+    enabled_namespaces: ""
+    secrets:
+      kube_system:
+        rgw_secret: kubernetes-etcd-backup-user
+        conf_secret: etcd-backup-restore
  etcdctlfilter:
    Filters:
      # etcdctl-rootwrap command filters for etcdctl utility container
      # This file should be owned by (and only-writeable by) the root user
+      dbutils: CommandFilter, dbutils, nobody
      etcdctl: CommandFilter, etcdctl, root
+      kubectl: CommandFilter, kubectl, root
  etcdctlrootwrapconf:
    DEFAULT:
      # Configuration for etcdctl-rootwrap
@ -201,4 +218,4 @@ manifests:
  configmap_etc_sudoers: true
  deployment_etcdctl_utility: true
  job_image_repo_sync: false
-  secret_certificates: true
+  secret_certificates: true
--- a/images/etcdctl-utility/Dockerfile.ubuntu_bionic
+++ b/images/etcdctl-utility/Dockerfile.ubuntu_bionic
@ -9,6 +9,7 @@ LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc
      org.opencontainers.image.licenses='Apache-2.0'

 ARG ETCDCTL_VERSION=3.4.2
+ARG KUBE_VERSION=1.17.3

 ENV GOOGLE_URL=https://storage.googleapis.com/etcd
 ENV GITHUB_URL=https://github.com/etcd-io/etcd/releases/download
@ -33,10 +34,15 @@ RUN set -xe \
       python3-pip \
    && pip3 install \
         oslo.rootwrap==5.8.0 \
+         python-openstackclient==3.18.1 \
+         python-swiftclient \
    && TMP_DIR=$(mktemp --directory) && cd ${TMP_DIR} \
    && curl -sSLO "${DOWNLOAD_URL}/v${ETCDCTL_VERSION}/etcd-v${ETCDCTL_VERSION}-linux-amd64.tar.gz" \
    && tar -xvf ${TMP_DIR}/etcd-v${ETCDCTL_VERSION}-linux-amd64.tar.gz --strip-components=1 \
        -C /usr/local/bin etcd-v${ETCDCTL_VERSION}-linux-amd64/etcdctl \
+    && curl -sSL https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 \
+    && mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl \
+    && chmod +x /usr/bin/kubectl \
    && apt-get clean -y \
    && rm -rf \
       /var/lib/apt/lists/* \
@ -47,4 +53,4 @@ RUN sed -i "/rootwrap_logger.setLevel/s/.*/#&/" /usr/local/lib/python3.6/dist-pa
    && sed -i "/os.path.basename/s/.*/#&/" /usr/local/lib/python3.6/dist-packages/oslo_rootwrap/wrapper.py \
    && sed -i "/rootwrap_logger.addHandler/s/.*/#&/" /usr/local/lib/python3.6/dist-packages/oslo_rootwrap/wrapper.py

-CMD ["/bin/bash"]
+CMD ["/bin/bash"]