Add host paths and openstack/swift clients to etcdctl utility

1) Adds etcd on demand job for backup and restore.
2) Adds dbutils implementation to perform manual local and
   remote backup/restore for etcd.
3) Adds the openstack and swift clients to the etcdctl utlity pod so
   that this pod can access etcd remote gateway backup files via the swift interface.
4) Adds kubectl to create backup/restore etcd jobs.

Change-Id: Iadfaa828366bae3a98552891c24d669f2922e1d1

charts/etcdctl-utility/templates/bin/utility/_etcd_ondemand_job.sh.tpl

@@ -0,0 +1,177 @@
+#!/bin/bash
+
+{{- $envAll := . }}
+
+export ETCD_POD_NAMESPACE=$1
+if [[ $ETCD_POD_NAMESPACE == "" ]]; then
+  echo "No namespace given - cannot spawn ondemand job."
+  exit 1
+fi
+
+export ETCD_RGW_SECRET={{ $envAll.Values.conf.etcd_backup_restore.secrets.kube_system.rgw_secret }}
+export ETCD_CONF_SECRET={{ $envAll.Values.conf.etcd_backup_restore.secrets.kube_system.conf_secret }}
+export ETCD_IMAGE_NAME=$(kubectl get cronjob -n ${ETCD_POD_NAMESPACE} kubernetes-etcd-backup -o yaml -o jsonpath="{range .spec.jobTemplate.spec.template.spec.containers[*]}{.image}{'\n'}{end}" | grep etcdctl-utility)
+export ETCD_BACKUP_BASE_PATH=$(kubectl get secret -o yaml -n ${ETCD_POD_NAMESPACE} ${ETCD_CONF_SECRET} | grep BACKUP_BASE_PATH | awk '{print $2}' | base64 -d)
+
+if [[ $NODE == "" ]];then
+  echo "Cannot find node to run ondemand job from."
+  exit 1
+fi
+
+if [[ $ETCD_IMAGE_NAME == "" ]]; then
+  echo "Cannot find the utility image for populating ETCD_IMAGE_NAME variable."
+  exit 1
+fi
+
+cat <<EOF | kubectl create -n $ETCD_POD_NAMESPACE -f -
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: etcd-ondemand
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+  labels:
+{{ tuple $envAll "etcd-ondemand" "ondemand" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "etcd-ondemand" "ondemand" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      restartPolicy: OnFailure
+      serviceAccountName: kubernetes-etcd-etcd-backup
+      nodeName: ${NODE}
+      containers:
+        - name: etcd-ondemand
+          image: ${ETCD_IMAGE_NAME}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.etcd_ondemand | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /bin/sleep
+            - "1000000"
+          env:
+            - name: ETCDCTL_API
+              value: "{{ .Values.conf.etcd.etcdctl_api }}"
+            - name: ETCDCTL_DIAL_TIMEOUT
+              value: 10s
+            - name: ETCDCTL_CACERT
+              value: /etc/etcd/tls/certs/client-ca.pem
+            - name: ETCDCTL_CERT
+              value: /etc/etcd/tls/certs/anchor-etcd-client.pem
+            - name: ETCDCTL_KEY
+              value: /etc/etcd/tls/keys/anchor-etcd-client-key.pem
+            - name: ETCDCTL_ENDPOINTS
+              value: https://{{ .Values.conf.etcd.endpoints }}:{{ .Values.endpoints.etcd.port.client.default }}
+            - name: ONDEMAND_JOB
+              value: etcd-ondemand
+            - name: ARCHIVE_DIR
+              value: $ETCD_BACKUP_BASE_PATH/db/$ETCD_POD_NAMESPACE/etcd/archive
+            - name: BACKUP_RESTORE_SCOPE
+              value: etcd
+            - name: BACKUP_RESTORE_NAMESPACE_LIST
+              value: $ETCD_POD_NAMESPACE
+            - name: ETCD_BACKUP_BASE_DIR
+              valueFrom:
+                secretKeyRef:
+                  key: BACKUP_BASE_PATH
+                  name: ${ETCD_CONF_SECRET}
+            - name: POD_NAMESPACE
+              value: ${ETCD_POD_NAMESPACE}
+            - name: REMOTE_BACKUP_ENABLED
+              valueFrom:
+                secretKeyRef:
+                  key: REMOTE_BACKUP_ENABLED
+                  name: ${ETCD_CONF_SECRET}
+            - name: CONTAINER_NAME
+              valueFrom:
+                secretKeyRef:
+                  key: REMOTE_BACKUP_CONTAINER
+                  name: ${ETCD_CONF_SECRET}
+            - name: OS_IDENTITY_API_VERSION
+              value: "3"
+            - name: OS_AUTH_URL
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_AUTH_URL
+            - name: OS_REGION_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_REGION_NAME
+            - name: OS_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_USERNAME
+            - name: OS_PROJECT_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_PROJECT_NAME
+            - name: OS_USER_DOMAIN_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_USER_DOMAIN_NAME
+            - name: OS_PROJECT_DOMAIN_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_PROJECT_DOMAIN_NAME
+            - name: OS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: ${ETCD_RGW_SECRET}
+                  key: OS_PASSWORD
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: kubernetes-etcd-certs
+              mountPath: /etc/etcd/tls/certs
+            - name: kubernetes-etcd-keys
+              mountPath: /etc/etcd/tls/keys
+            - mountPath: /tmp/restore_etcd.sh
+              name: kubernetes-etcd-bin
+              readOnly: true
+              subPath: restore_etcd.sh
+            - mountPath: /tmp/restore_main.sh
+              name: kubernetes-etcd-bin
+              readOnly: true
+              subPath: restore_main.sh
+            - mountPath: /tmp/backup_etcd.sh
+              name: kubernetes-etcd-bin
+              readOnly: true
+              subPath: backup_etcd.sh
+            - mountPath: /tmp/bin/backup_main.sh
+              name: kubernetes-etcd-bin
+              readOnly: true
+              subPath: backup_main.sh
+            - mountPath: {{ .Values.conf.backup.host_backup_path }}
+              name: kubernetes-etcd-backup-dir
+              subPath: .
+            - name: host-etcd
+              mountPath: /var/lib/etcd
+              subPath: .
+      volumes:
+        - name: pod-tmp
+          emptyDir: {}
+        - name: kubernetes-etcd-certs
+          configMap:
+            name: kubernetes-etcd-certs
+            defaultMode: 0444
+        - name: kubernetes-etcd-keys
+          secret:
+            secretName: kubernetes-etcd-keys
+            defaultMode: 0444
+        - name: kubernetes-etcd-bin
+          configMap:
+            name: kubernetes-etcd-bin
+            defaultMode: 0555
+        - name: kubernetes-etcd-backup-dir
+          hostPath:
+            path: {{ .Values.conf.backup.host_backup_path }}
+        - name: host-etcd
+          hostPath:
+            path: /var/lib/etcd
+EOF

charts/etcdctl-utility/templates/configmap-bin.yaml

@@ -39,6 +39,12 @@ data:
   utilscli: |
 {{ tuple "bin/utility/_utilscli.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
 
+  etcd-ondemand-job.sh: |
+{{ tuple "bin/utility/_etcd_ondemand_job.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
+  dbutils: |
+{{ tuple "bin/utility/_dbutils.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
 {{- if .Values.bootstrap.enabled }}
   bootstrap.sh: |
 {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

charts/etcdctl-utility/templates/deployment-etcdctl-utility.yaml

@@ -27,6 +27,7 @@ rules:
       - namespaces
       - nodes
       - pods
+      - pods/exec
       - persistentvolumeclaims
       - persistentvolumes
     verbs:
@@ -34,6 +35,33 @@ rules:
       - list
       - watch
       - create
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "batch"
+    resources:
+      - cronjobs
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "batch"
+    resources:
+      - jobs
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
@@ -96,6 +124,10 @@ spec:
               value: /etc/kubernetes/apiserver/pki/etcd-client-key.pem
             - name: ETCDCTL_ENDPOINTS
               value: https://{{ .Values.conf.etcd.endpoints }}:{{ .Values.endpoints.etcd.port.client.default }}
+            - name: BACKUP_RESTORE_NAMESPACE_LIST
+              value: {{ .Values.conf.etcd_backup_restore.enabled_namespaces | quote }}
+            - name: BACKUP_RESTORE_SCOPE
+              value: "etcd"
           command:
             - /tmp/bootstrap.sh
           volumeMounts:
@@ -127,6 +159,14 @@ spec:
               mountPath: /usr/local/bin/etcdctl-utility-rootwrap
               subPath: etcdctl-utility-rootwrap
               readOnly: true
+            - name: etcdctl-utility-bin-utilscli
+              mountPath: /usr/local/bin/dbutils
+              subPath: dbutils
+              readOnly: true
+            - name: etcdctl-utility-bin-utilscli
+              mountPath: /tmp/etcd-ondemand-job.sh
+              subPath: etcd-ondemand-job.sh
+              readOnly: true
             - name: etcdctl-utility-sudoers
               mountPath: /etc/sudoers.d/utilscli-sudo
               subPath: utilscli-sudo

charts/etcdctl-utility/values.yaml

@@ -83,6 +83,13 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+      etcd_ondemand:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
     test:
       etcdctl:
         requests:
@@ -92,11 +99,21 @@ pod:
           memory: "1024Mi"
           cpu: "2000m"
 conf:
+  backup:
+    host_backup_path: /var/backups
+  etcd_backup_restore:
+    enabled_namespaces: ""
+    secrets:
+      kube_system:
+        rgw_secret: kubernetes-etcd-backup-user
+        conf_secret: etcd-backup-restore
   etcdctlfilter:
     Filters:
       # etcdctl-rootwrap command filters for etcdctl utility container
       # This file should be owned by (and only-writeable by) the root user
+      dbutils: CommandFilter, dbutils, nobody
       etcdctl: CommandFilter, etcdctl, root
+      kubectl: CommandFilter, kubectl, root
   etcdctlrootwrapconf:
     DEFAULT:
       # Configuration for etcdctl-rootwrap
@@ -201,4 +218,4 @@ manifests:
   configmap_etc_sudoers: true
   deployment_etcdctl_utility: true
   job_image_repo_sync: false
-  secret_certificates: true
+  secret_certificates: true

images/etcdctl-utility/Dockerfile.ubuntu_bionic

@@ -9,6 +9,7 @@ LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc
       org.opencontainers.image.licenses='Apache-2.0'
 
 ARG ETCDCTL_VERSION=3.4.2
+ARG KUBE_VERSION=1.17.3
 
 ENV GOOGLE_URL=https://storage.googleapis.com/etcd
 ENV GITHUB_URL=https://github.com/etcd-io/etcd/releases/download
@@ -33,10 +34,15 @@ RUN set -xe \
        python3-pip \
     && pip3 install \
          oslo.rootwrap==5.8.0 \
+         python-openstackclient==3.18.1 \
+         python-swiftclient \
     && TMP_DIR=$(mktemp --directory) && cd ${TMP_DIR} \
     && curl -sSLO "${DOWNLOAD_URL}/v${ETCDCTL_VERSION}/etcd-v${ETCDCTL_VERSION}-linux-amd64.tar.gz" \
     && tar -xvf ${TMP_DIR}/etcd-v${ETCDCTL_VERSION}-linux-amd64.tar.gz --strip-components=1 \
         -C /usr/local/bin etcd-v${ETCDCTL_VERSION}-linux-amd64/etcdctl \
+    && curl -sSL https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 \
+    && mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl \
+    && chmod +x /usr/bin/kubectl \
     && apt-get clean -y \
     && rm -rf \
        /var/lib/apt/lists/* \
@@ -47,4 +53,4 @@ RUN sed -i "/rootwrap_logger.setLevel/s/.*/#&/" /usr/local/lib/python3.6/dist-pa
     && sed -i "/os.path.basename/s/.*/#&/" /usr/local/lib/python3.6/dist-packages/oslo_rootwrap/wrapper.py \
     && sed -i "/rootwrap_logger.addHandler/s/.*/#&/" /usr/local/lib/python3.6/dist-packages/oslo_rootwrap/wrapper.py
 
-CMD ["/bin/bash"]
+CMD ["/bin/bash"]