[etcd] Switch etcd to staetefulset

* Switch etcd to statefulset
* Allow to use persistant volumes to store etcd data
* Allow to deploy in clustered mode

Change-Id: I2baf5bdd05c280067991bb8b7f00c887ffd95c20

etcd/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v3.4.3
 description: OpenStack-Helm etcd
 name: etcd
-version: 0.1.7
+version: 0.1.8
 home: https://coreos.com/etcd/
 icon: https://raw.githubusercontent.com/CloudCoreo/etcd-cluster/master/images/icon.png
 sources:

etcd/templates/bin/_etcd-healthcheck.sh.tpl

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -x
+
+export ETCDCTL_API=3
+
+ETCD_CLIENT_PORT={{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+DISCOVERY_DOMAIN={{ tuple "etcd" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+
+etcdctl endpoint health --endpoints=${POD_NAME}.${DISCOVERY_DOMAIN}:${ETCD_CLIENT_PORT}

etcd/templates/bin/_etcd.sh.tpl

@@ -16,6 +16,60 @@ limitations under the License.
 
 set -ex
 
+active_members_present() {
+  res=1
+  for endpoint in $(echo $ETCD_ENDPOINTS | tr ',' '\n'); do
+      if etcdctl endpoint health --endpoints=$endpoint >/dev/null 2>&1; then
+          res=$?
+          if [[ "$res" == 0 ]]; then
+              break
+          fi
+      fi
+  done
+  echo $res
+}
+
+ETCD_REPLICAS={{ .Values.pod.replicas.etcd }}
+PEER_PREFIX_NAME={{- printf "%s-%s" .Release.Name "etcd"  }}
+DISCOVERY_DOMAIN={{ tuple "etcd" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+ETCD_PEER_PORT=2380
+ETCD_CLIENT_PORT={{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+ETCD_PROTOCOL={{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
+PEERS="${PEER_PREFIX_NAME}-0=${ETCD_PROTOCOL}://${PEER_PREFIX_NAME}-0.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}"
+ETCD_ENDPOINTS="${ETCD_PROTOCOL}://${PEER_PREFIX_NAME}-0.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}"
+if [[ ${ETCD_REPLICAS} -gt 1 ]] ; then
+  for i in $(seq 1 $(( ETCD_REPLICAS - 1 ))); do
+    PEERS="$PEERS,${PEER_PREFIX_NAME}-${i}=${ETCD_PROTOCOL}://${PEER_PREFIX_NAME}-${i}.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}"
+    ETCD_ENDPOINTS="${ETCD_ENDPOINTS},${ETCD_PROTOCOL}://${PEER_PREFIX_NAME}-${i}.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}"
+  done
+fi
+ADVERTISE_PEER_URL="${ETCD_PROTOCOL}://${HOSTNAME}.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}"
+ADVERTISE_CLIENT_URL="${ETCD_PROTOCOL}://${HOSTNAME}.${DISCOVERY_DOMAIN}:${ETCD_CLIENT_PORT}"
+
+ETCD_INITIAL_CLUSTER_STATE=new
+
+if [[ -z "$(ls -A $ETCD_DATA_DIR)" ]]; then
+  echo "State directory $ETCD_DATA_DIR is empty."
+  if [[ $(active_members_present) -eq 0 ]]; then
+      ETCD_INITIAL_CLUSTER_STATE=existing
+      member_id=$(etcdctl --endpoints=${ETCD_ENDPOINTS} member list | grep -w ${ADVERTISE_CLIENT_URL} | awk -F "," '{ print $1 }')
+      if [[ -n "$member_id" ]]; then
+          echo "Current node is a member of cluster, member_id: ${member_id}"
+          echo "Rejoining..."
+          echo "Removing member from the cluster"
+          etcdctl member remove "$member_id" --endpoints=${ETCD_ENDPOINTS}
+          etcdctl member add ${ADVERTISE_CLIENT_URL} --peer-urls=${ADVERTISE_PEER_URL} --endpoints=${ETCD_ENDPOINTS}
+      fi
+  else
+      echo "Do not have active members. Starting initial cluster state."
+  fi
+fi
+
 exec etcd \
-  --listen-client-urls http://0.0.0.0:{{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} \
+  --name ${HOSTNAME} \
-  --advertise-client-urls {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix "/" }}
+  --listen-peer-urls ${ETCD_PROTOCOL}://0.0.0.0:${ETCD_PEER_PORT} \
+  --listen-client-urls ${ETCD_PROTOCOL}://0.0.0.0:${ETCD_CLIENT_PORT} \
+  --advertise-client-urls ${ADVERTISE_CLIENT_URL} \
+  --initial-advertise-peer-urls ${ADVERTISE_PEER_URL} \
+  --initial-cluster ${PEERS} \
+  --initial-cluster-state ${ETCD_INITIAL_CLUSTER_STATE}

etcd/templates/configmap-bin.yaml

@@ -27,4 +27,6 @@ data:
 {{- end }}
   etcd.sh: |
 {{ tuple "bin/_etcd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  etcd-healthcheck.sh: |
+{{ tuple "bin/_etcd-healthcheck.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}

etcd/templates/service-discovery.yaml

@@ -0,0 +1,34 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if .Values.manifests.service_discovery }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tuple "etcd" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+spec:
+  ports:
+    - name: client
+      port: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+      protocol: TCP
+      targetPort: client
+    - name: peer
+      port: {{ tuple "etcd_discovery" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+      protocol: TCP
+      targetPort: peer
+  publishNotReadyAddresses: true
+  clusterIP: None
+  selector:
+{{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- end }}

etcd/templates/service.yaml

@@ -20,7 +20,14 @@ metadata:
 spec:
   sessionAffinity: ClientIP
   ports:
-    - port: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    - name: client
+      port: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+      protocol: TCP
+      targetPort: client
+    - name: peer
+      port: {{ tuple "etcd_discovery" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+      protocol: TCP
+      targetPort: peer
   selector:
 {{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 {{- end }}

etcd/templates/statefulset.yaml

@@ -10,7 +10,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-{{- if .Values.manifests.deployment }}
+{{- define "etcdProbeTemplate" }}
+exec:
+  command:
+    - /tmp/etcd-healthcheck.sh
+{{- end }}
+
+{{- if .Values.manifests.statefulset }}
 {{- $envAll := . }}
 
 {{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "etcd"  }}
@@ -19,7 +25,7 @@
 {{ tuple $envAll "etcd" $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: {{ $rcControllerName | quote }}
   annotations:
@@ -27,11 +33,12 @@ metadata:
   labels:
 {{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 spec:
+  podManagementPolicy: "Parallel"
+  serviceName: "{{ tuple "etcd" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}"
   replicas: {{ .Values.pod.replicas.etcd }}
   selector:
     matchLabels:
 {{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
-{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
   template:
     metadata:
       labels:
@@ -41,8 +48,8 @@ spec:
 {{ dict "envAll" $envAll "podName" "etcd" "containerNames" (list "init" "etcd") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }}
         configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
     spec:
-{{ dict "envAll" $envAll "application" "etcd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       serviceAccountName: {{ $rcControllerName | quote }}
+{{ dict "envAll" $envAll "application" "etcd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
       affinity:
 {{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
       nodeSelector:
@@ -53,13 +60,24 @@ spec:
         - name: etcd
 {{ tuple $envAll "etcd" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ dict "envAll" $envAll "application" "etcd" "container" "etcd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{ dict "envAll" . "component" "etcd" "container" "etcd" "type" "readiness" "probeTemplate" (include "etcdProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
+{{ dict "envAll" . "component" "etcd" "container" "etcd" "type" "liveness" "probeTemplate" (include "etcdProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
+          env:
+{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.etcd | indent 12 }}
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
           command:
             - /tmp/etcd.sh
           ports:
             - containerPort: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
-          readinessProbe:
+              name: client
-            tcpSocket:
+              protocol: TCP
-              port: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - containerPort: {{ tuple "etcd_discovery" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+              name: peer
+              protocol: TCP
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
@@ -67,6 +85,12 @@ spec:
               mountPath: /tmp/etcd.sh
               subPath: etcd.sh
               readOnly: true
+            - name: etcd-data
+              mountPath: /var/lib/etcd
+            - name: etcd-bin
+              mountPath: /tmp/etcd-healthcheck.sh
+              subPath: etcd-healthcheck.sh
+              readOnly: true
       volumes:
         - name: pod-tmp
           emptyDir: {}
@@ -74,4 +98,21 @@ spec:
           configMap:
             name: {{ $configMapBinName | quote }}
             defaultMode: 0555
+        {{- if not .Values.volume.enabled }}
+        - name: etcd-data
+          emptyDir: {}
+        {{- end }}
+{{- end }}
+{{- if .Values.volume.enabled }}
+  volumeClaimTemplates:
+  - metadata:
+      name: etcd-data
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: {{ .Values.volume.size }}
+      {{- if ne .Values.volume.class_name "default" }}
+      storageClassName: {{ .Values.volume.class_name }}
+      {{- end }}
 {{- end }}

etcd/values.yaml

@@ -51,6 +51,10 @@ dependencies:
       jobs: null
 
 pod:
+  env:
+    etcd:
+      ETCD_DATA_DIR: /var/lib/etcd
+      ETCD_INITIAL_CLUSTER_TOKEN: etcd-cluster-1
   security_context:
     etcd:
       pod:
@@ -64,6 +68,21 @@ pod:
     etcd:
       init: runtime/default
       etcd: runtime/default
+  probes:
+    etcd:
+      etcd:
+        readiness:
+          enabled: True
+          params:
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 1
+        liveness:
+          enabled: True
+          params:
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 1
   affinity:
     anti:
       type:
@@ -129,6 +148,7 @@ endpoints:
     name: etcd
     hosts:
       default: etcd
+      discovery: etcd-discovery
     host_fqdn_override:
       default: null
     path:
@@ -138,11 +158,30 @@ endpoints:
     port:
       client:
         default: 2379
+  etcd_discovery:
+    name: etcd-discovery
+    hosts:
+      default: etcd-discovery
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: 'http'
+    port:
+      client:
+        default: 2380
+
+volume:
+  enabled: false
+  class_name: general
+  size: 5Gi
 
 manifests:
   configmap_bin: true
-  deployment: true
+  statefulset: true
   job_image_repo_sync: true
   secret_registry: true
   service: true
+  service_discovery: true
 ...

releasenotes/notes/etcd.yaml

@@ -8,4 +8,5 @@ etcd:
   - 0.1.5 Added OCI registry authentication
   - 0.1.6 Update kubernetes registry to registry.k8s.io
   - 0.1.7 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default
+  - 0.1.8 Switch etcd to staetefulset
 ...