Enable TLS between Elasticsearch and Kibana

This change enables TLS between Elasticsearch and Kibana
data path. Note that TLS terminates at apache-proxy container
of the Elasticsearch-client pod, not directly to port 9200 of
elasticsearch-client container.

Since all data traffic goes through apache-proxy container,
fluentd output to Elasticsearch are configured to have TLS
enabled as well.

In additon, other Elasticsearch pods that communicate with
Elasticsearch-client endpoint are modified to provide
the cacert option with curl.

Change-Id: I3373c0c350b30c175be4a34d25a403b9caf74294

elasticsearch/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v7.6.2
 description: OpenStack-Helm ElasticSearch
 name: elasticsearch
-version: 0.2.4
+version: 0.2.5
 home: https://www.elastic.co/
 sources:
   - https://github.com/elastic/elasticsearch

elasticsearch/templates/bin/_create_s3_buckets.sh.tpl

@@ -53,6 +53,8 @@ RGW_PROTO={{ $client.settings.protocol | default (tuple "ceph_object_store" "int
 CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST"
 if [ "$RGW_PROTO" = "http" ]; then
   CONNECTION_ARGS+=" --no-ssl"
+else
+  CONNECTION_ARGS+=" --no-check-certificate"
 fi
 
 USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY"

elasticsearch/templates/bin/_create_template.sh.tpl

@@ -21,9 +21,9 @@ NUM_ERRORS=0
 {{ if not (empty $object) }}
 
 echo "creating {{$name}}"
-error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+error=$(curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
    -X{{ $object.method | default "PUT" | upper }} \
-   "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/{{ $object.endpoint }}" \
+   "${ELASTICSEARCH_ENDPOINT}/{{ $object.endpoint }}" \
    -H 'Content-Type: application/json' -d '{{ $object.body | toJson }}' | jq -r '.error')
 
 if [ $error == "null" ]; then

elasticsearch/templates/bin/_elasticsearch.sh.tpl

@@ -45,11 +45,11 @@ function stop () {
 function wait_to_join() {
   # delay 5 seconds before the first check
   sleep 5
-  joined=$(curl -s -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/_cat/nodes" | grep -w $NODE_NAME || true )
+  joined=$(curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/_cat/nodes" | grep -w $NODE_NAME || true )
   i=0
   while [ -z "$joined" ]; do
     sleep 5
-    joined=$(curl -s -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/_cat/nodes" | grep -w $NODE_NAME || true )
+    joined=$(curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/_cat/nodes" | grep -w $NODE_NAME || true )
     i=$((i+1))
     # Waiting for up to 60 minutes
     if [ $i -gt 720 ]; then
@@ -62,7 +62,7 @@ function allocate_data_node () {
   echo "Node ${NODE_NAME} has started. Waiting to rejoin the cluster."
   wait_to_join
   echo "Re-enabling Replica Shard Allocation"
-  curl -s -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPUT -H 'Content-Type: application/json' \
+  curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPUT -H 'Content-Type: application/json' \
     "${ELASTICSEARCH_ENDPOINT}/_cluster/settings" -d "{
     \"persistent\": {
       \"cluster.routing.allocation.enable\": null
@@ -102,7 +102,7 @@ function start_data_node () {
     # https://www.elastic.co/guide/en/elasticsearch/reference/7.x/restart-cluster.html#restart-cluster-rolling
 
     echo "Disabling Replica Shard Allocation"
-    curl -s -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPUT -H 'Content-Type: application/json' \
+    curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPUT -H 'Content-Type: application/json' \
       "${ELASTICSEARCH_ENDPOINT}/_cluster/settings" -d "{
       \"persistent\": {
         \"cluster.routing.allocation.enable\": \"primaries\"
@@ -112,7 +112,7 @@ function start_data_node () {
     # If version < 7.6 use _flush/synced; otherwise use _flush
     # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-synced-flush-api.html#indices-synced-flush-api
 
-    version=$(curl -s -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/" | jq -r .version.number)
+    version=$(curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/" | jq -r .version.number)
 
     if [[ $version =~ "7.1" ]]; then
       action="_flush/synced"
@@ -120,7 +120,7 @@ function start_data_node () {
       action="_flush"
     fi
 
-    curl -s -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPOST "${ELASTICSEARCH_ENDPOINT}/$action"
+    curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPOST "${ELASTICSEARCH_ENDPOINT}/$action"
 
     # TODO: Check the response of synced flush operations to make sure there are no failures.
     # Synced flush operations that fail due to pending indexing operations are listed in the response body,

elasticsearch/templates/bin/_helm-tests.sh.tpl

@@ -16,7 +16,7 @@ limitations under the License.
 set -ex
 
 function create_test_index () {
-  index_result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  index_result=$(curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
   -XPUT "${ELASTICSEARCH_ENDPOINT}/test_index?pretty" -H 'Content-Type: application/json' -d'
   {
     "settings" : {
@@ -38,13 +38,13 @@ function create_test_index () {
 
 {{ if .Values.conf.elasticsearch.snapshots.enabled }}
 function check_snapshot_repositories_verified () {
-  repositories=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  repositories=$(curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
                   "${ELASTICSEARCH_ENDPOINT}/_snapshot" | jq -r "keys | @sh" )
 
   repositories=$(echo $repositories | sed "s/'//g") # Strip single quotes from jq output
 
   for repository in $repositories; do
-    error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+    error=$(curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
             -XPOST "${ELASTICSEARCH_ENDPOINT}/_snapshot/${repository}/_verify" | jq -r '.error')
 
     if [ $error == "null" ]; then
@@ -59,7 +59,7 @@ function check_snapshot_repositories_verified () {
 
 function remove_test_index () {
   echo "Deleting index created for service testing"
-  curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
   -XDELETE "${ELASTICSEARCH_ENDPOINT}/test_index"
 }
 

elasticsearch/templates/bin/_verify-repositories.sh.tpl

@@ -18,12 +18,12 @@ limitations under the License.
 set -ex
 
 function verify_snapshot_repository() {
-  curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-    -XPOST "${ELASTICSEARCH_HOST}/_snapshot/$1/_verify"
+    -XPOST "${ELASTICSEARCH_ENDPOINT}/_snapshot/$1/_verify"
 }
 
-repositories=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+repositories=$(curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-                "${ELASTICSEARCH_HOST}/_snapshot" | jq -r 'keys | @sh')
+                "${ELASTICSEARCH_ENDPOINT}/_snapshot" | jq -r 'keys | @sh')
 
 repositories=$(echo $repositories | sed "s/'//g") # Strip single quotes from jq output
 

elasticsearch/templates/certificates.yaml

@@ -0,0 +1,17 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.certificates -}}
+{{  dict "envAll" . "service" "elasticsearch" "type" "internal" | include "helm-toolkit.manifests.certificates" }}
+{{- end -}}

elasticsearch/templates/cron-job-verify-repositories.yaml

@@ -70,8 +70,12 @@ spec:
                     secretKeyRef:
                       name: {{ $esUserSecret }}
                       key: ELASTICSEARCH_PASSWORD
-                - name: ELASTICSEARCH_HOST
+                - name: ELASTICSEARCH_ENDPOINT
-                  value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+                  value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }}
+{{- if .Values.manifests.certificates }}
+                - name: CACERT_OPTION
+                  value: "--cacert /etc/elasticsearch/certs/ca.crt"
+{{- end }}
               volumeMounts:
                 - name: pod-tmp
                   mountPath: /tmp
@@ -79,6 +83,7 @@ spec:
                   mountPath: /tmp/verify-repositories.sh
                   subPath: verify-repositories.sh
                   readOnly: true
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }}
           volumes:
             - name: pod-tmp
               emptyDir: {}
@@ -86,4 +91,5 @@ spec:
               configMap:
                 name: elasticsearch-bin
                 defaultMode: 0555
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 12 }}
 {{- end }}

elasticsearch/templates/deployment-client.yaml

@@ -12,6 +12,20 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- define "probeTemplate" }}
+{{- $probePort := tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $probeUser := .Values.endpoints.elasticsearch.auth.admin.username }}
+{{- $probePass := .Values.endpoints.elasticsearch.auth.admin.password }}
+{{- $authHeader := printf "%s:%s" $probeUser $probePass | b64enc }}
+httpGet:
+  path: /_cluster/health
+  scheme: {{ tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }}
+  port: {{ $probePort }}
+  httpHeaders:
+    - name: Authorization
+      value: Basic {{ $authHeader }}
+{{- end }}
+
 {{- if .Values.manifests.deployment_client }}
 {{- $envAll := . }}
 
@@ -73,7 +87,7 @@ spec:
             - /tmp/apache.sh
             - start
           ports:
-            - name: http
+            - name: {{ tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
               containerPort: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           readinessProbe:
             tcpSocket:
@@ -112,6 +126,7 @@ spec:
               mountPath: /usr/local/apache2/conf/httpd.conf
               subPath: httpd.conf
               readOnly: true
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
         - name: elasticsearch-client
 {{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.client | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@@ -126,8 +141,6 @@ spec:
                   - /tmp/elasticsearch.sh
                   - stop
           ports:
-            - name: http
-              containerPort: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
             - name: transport
               containerPort: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           livenessProbe:
@@ -135,10 +148,7 @@ spec:
               port: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
             initialDelaySeconds: 20
             periodSeconds: 10
-          readinessProbe:
+{{ dict "envAll" . "component" "elasticsearch" "container" "elasticsearch-client" "type" "readiness" "probeTemplate" (include "probeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
-            httpGet:
-              path: /_cluster/health
-              port: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           env:
             - name: NAMESPACE
               valueFrom:
@@ -210,5 +220,6 @@ spec:
             defaultMode: 0444
         - name: storage
           emptyDir: {}
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
 {{- end }}

elasticsearch/templates/ingress-elasticsearch.yaml

@@ -13,6 +13,12 @@ limitations under the License.
 */}}
 
 {{- if and .Values.manifests.ingress .Values.network.elasticsearch.ingress.public }}
-{{- $ingressOpts := dict "envAll" . "backendService" "elasticsearch" "backendServiceType" "elasticsearch" "backendPort" "http" -}}
+{{- $envAll := . -}}
+{{- $port := tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
+{{- $ingressOpts := dict "envAll" $envAll "backendService" "elasticsearch" "backendServiceType" "elasticsearch" "backendPort" $port -}}
+{{- $secretName := $envAll.Values.secrets.tls.elasticsearch.elasticsearch.internal -}}
+{{- if and .Values.manifests.certificates $secretName -}}
+{{- $_ := set $ingressOpts "certIssuer" .Values.endpoints.elasticsearch.host_fqdn_override.default.tls.issuerRef.name -}}
+{{- end -}}
 {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
 {{- end }}

elasticsearch/templates/job-elasticsearch-template.yaml

@@ -50,10 +50,12 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_templates | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "create_template" "container" "create_elasticsearch_template" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           env:
-            - name: ELASTICSEARCH_HOST
+            - name: ELASTICSEARCH_ENDPOINT
-              value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
+              value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }}
-            - name: ELASTICSEARCH_PORT
+{{- if .Values.manifests.certificates }}
-              value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: CACERT_OPTION
+              value: "--cacert /etc/elasticsearch/certs/ca.crt"
+{{- end }}
             - name: ELASTICSEARCH_USERNAME
               valueFrom:
                 secretKeyRef:
@@ -73,6 +75,7 @@ spec:
               mountPath: /tmp/create_template.sh
               subPath: create_template.sh
               readOnly: true
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
 {{ if $mounts_elasticsearch_templates.volumeMounts }}{{ toYaml $mounts_elasticsearch_templates.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: pod-tmp
@@ -81,5 +84,6 @@ spec:
           configMap:
             name: elasticsearch-bin
             defaultMode: 0555
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{ if $mounts_elasticsearch_templates.volumes }}{{ toYaml $mounts_elasticsearch_templates.volumes | indent 8 }}{{ end }}
 {{- end }}

elasticsearch/templates/pod-helm-tests.yaml

@@ -56,7 +56,11 @@ spec:
               name: {{ $esUserSecret }}
               key: ELASTICSEARCH_PASSWORD
         - name: ELASTICSEARCH_ENDPOINT
-          value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+          value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }}
+{{- if .Values.manifests.certificates }}
+        - name: CACERT_OPTION
+          value: "--cacert /etc/elasticsearch/certs/ca.crt"
+{{- end }}
       volumeMounts:
         - name: pod-tmp
           mountPath: /tmp
@@ -64,6 +68,7 @@ spec:
           mountPath: /tmp/helm-tests.sh
           subPath: helm-tests.sh
           readOnly: true
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 8 }}
   volumes:
     - name: pod-tmp
       emptyDir: {}
@@ -71,4 +76,5 @@ spec:
       configMap:
         name: elasticsearch-bin
         defaultMode: 0555
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 4 }}
 {{- end }}

elasticsearch/templates/secret-elasticsearch.yaml

@@ -19,7 +19,8 @@ limitations under the License.
 {{- $elasticsearch_user := .Values.endpoints.elasticsearch.auth.admin.username }}
 {{- $elasticsearch_password := .Values.endpoints.elasticsearch.auth.admin.password }}
 {{- $elasticsearch_host := tuple "elasticsearch" "internal" "http" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
-{{- $elasticsearch_uri := printf "http://%s:%s@%s" $elasticsearch_user $elasticsearch_password $elasticsearch_host }}
+{{- $elasticsearch_scheme := tuple "elasticsearch" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
+{{- $elasticsearch_uri := printf "%s://%s:%s@%s" $elasticsearch_scheme $elasticsearch_user $elasticsearch_password $elasticsearch_host }}
 ---
 apiVersion: v1
 kind: Secret

elasticsearch/templates/service-logging.yaml

@@ -21,8 +21,9 @@ metadata:
   name: {{ tuple "elasticsearch" "default" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 spec:
   ports:
-  - name: http
+  - name: {{ tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
     port: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    targetPort: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
     {{- if .Values.network.elasticsearch.node_port.enabled }}
     nodePort: {{ .Values.network.elasticsearch.node_port.port }}
     {{- end }}

elasticsearch/templates/statefulset-data.yaml

@@ -114,7 +114,11 @@ spec:
                   name: {{ $esUserSecret }}
                   key: ELASTICSEARCH_PASSWORD
             - name: ELASTICSEARCH_ENDPOINT
-              value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+              value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }}
+{{- if .Values.manifests.certificates }}
+            - name: CACERT_OPTION
+              value: "--cacert /etc/elasticsearch/certs/ca.crt"
+{{- end }}
             - name: NODE_MASTER
               value: "false"
             - name: NODE_INGEST
@@ -158,6 +162,7 @@ spec:
               readOnly: true
             - name: storage
               mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
 {{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
       volumes:
         - name: pod-tmp
@@ -172,6 +177,7 @@ spec:
           secret:
             secretName: elasticsearch-etc
             defaultMode: 0444
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
 {{- if not .Values.storage.data.enabled }}
         - name: storage

elasticsearch/values.yaml

@@ -289,6 +289,14 @@ pod:
         timeout: 600
       prometheus_elasticsearch_exporter:
         timeout: 600
+  probes:
+    elasticsearch:
+      elasticsearch-client:
+        readiness:
+          enabled: true
+          params:
+            initialDelaySeconds: 30
+            timeoutSeconds: 30
   mounts:
     elasticsearch:
       elasticsearch:
@@ -418,6 +426,7 @@ secrets:
     elasticsearch:
       elasticsearch:
         public: elasticsearch-tls-public
+        internal: elasticsearch-tls-api
 
 jobs:
   curator:
@@ -788,13 +797,6 @@ endpoints:
       public: elasticsearch
     host_fqdn_override:
       default: null
-      # NOTE(srwilkers): this chart supports TLS for fqdn over-ridden public
-      # endpoints using the following format:
-      # public:
-      #   host: null
-      #   tls:
-      #     crt: null
-      #     key: null
     path:
       default: null
     scheme:
@@ -932,6 +934,7 @@ storage:
     #       - --region="default:backup"
 
 manifests:
+  certificates: false
   configmap_bin_curator: false
   configmap_bin_elasticsearch: true
   configmap_etc_curator: false

elasticsearch/values_overrides/tls.yaml

@@ -0,0 +1,138 @@
+---
+endpoints:
+  elasticsearch:
+    host_fqdn_override:
+      default:
+        tls:
+          secretName: elasticsearch-tls-api
+          issuerRef:
+            name: ca-issuer
+            kind: ClusterIssuer
+    scheme:
+      default: "https"
+    port:
+      http:
+        default: 443
+network:
+  elasticsearch:
+    ingress:
+      annotations:
+        nginx.ingress.kubernetes.io/backend-protocol: https
+conf:
+  httpd: |
+    ServerRoot "/usr/local/apache2"
+
+    Listen 443
+
+    LoadModule allowmethods_module modules/mod_allowmethods.so
+    LoadModule mpm_event_module modules/mod_mpm_event.so
+    LoadModule authn_file_module modules/mod_authn_file.so
+    LoadModule authn_core_module modules/mod_authn_core.so
+    LoadModule authz_host_module modules/mod_authz_host.so
+    LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
+    LoadModule authz_user_module modules/mod_authz_user.so
+    LoadModule authz_core_module modules/mod_authz_core.so
+    LoadModule access_compat_module modules/mod_access_compat.so
+    LoadModule auth_basic_module modules/mod_auth_basic.so
+    LoadModule ldap_module modules/mod_ldap.so
+    LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
+    LoadModule reqtimeout_module modules/mod_reqtimeout.so
+    LoadModule filter_module modules/mod_filter.so
+    LoadModule proxy_html_module modules/mod_proxy_html.so
+    LoadModule log_config_module modules/mod_log_config.so
+    LoadModule env_module modules/mod_env.so
+    LoadModule headers_module modules/mod_headers.so
+    LoadModule setenvif_module modules/mod_setenvif.so
+    LoadModule version_module modules/mod_version.so
+    LoadModule proxy_module modules/mod_proxy.so
+    LoadModule proxy_connect_module modules/mod_proxy_connect.so
+    LoadModule proxy_http_module modules/mod_proxy_http.so
+    LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
+    LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
+    LoadModule slotmem_plain_module modules/mod_slotmem_plain.so
+    LoadModule unixd_module modules/mod_unixd.so
+    LoadModule status_module modules/mod_status.so
+    LoadModule autoindex_module modules/mod_autoindex.so
+    LoadModule rewrite_module modules/mod_rewrite.so
+    LoadModule ssl_module modules/mod_ssl.so
+
+    <IfModule unixd_module>
+    User daemon
+    Group daemon
+    </IfModule>
+
+    <Directory />
+        AllowOverride none
+        Require all denied
+    </Directory>
+
+    <Files ".ht*">
+        Require all denied
+    </Files>
+
+    ErrorLog /dev/stderr
+
+    LogLevel warn
+
+    <IfModule log_config_module>
+        LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+        LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy
+        LogFormat "%h %l %u %t \"%r\" %>s %b" common
+
+        <IfModule logio_module>
+          LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
+        </IfModule>
+
+        SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
+        CustomLog /dev/stdout common
+        CustomLog /dev/stdout combined
+        CustomLog /dev/stdout proxy env=forwarded
+    </IfModule>
+
+    <Directory "/usr/local/apache2/cgi-bin">
+        AllowOverride None
+        Options None
+        Require all granted
+    </Directory>
+
+    <IfModule headers_module>
+        RequestHeader unset Proxy early
+    </IfModule>
+
+    <IfModule proxy_html_module>
+    Include conf/extra/proxy-html.conf
+    </IfModule>
+
+    <VirtualHost *:443>
+      <Location />
+          ProxyPass http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
+          ProxyPassReverse http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
+          AuthName "Elasticsearch"
+          AuthType Basic
+          AuthBasicProvider file ldap
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }}
+          AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }}
+          AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+          Require valid-user
+      </Location>
+
+      # Restrict access to the Elasticsearch Update By Query API Endpoint to prevent modification of indexed documents
+      <Location /*/_update_by_query*>
+          Require all denied
+      </Location>
+      # Restrict access to the Elasticsearch Delete By Query API Endpoint to prevent deletion of indexed documents
+      <Location /*/_delete_by_query*>
+          Require all denied
+      </Location>
+      SSLEngine On
+      SSLProxyEngine on
+      SSLCertificateFile      /etc/elasticsearch/certs/tls.crt
+      SSLCertificateKeyFile   /etc/elasticsearch/certs/tls.key
+      SSLProtocol             all -SSLv3 -TLSv1 -TLSv1.1
+      SSLCipherSuite          ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
+      SSLHonorCipherOrder     on
+    </VirtualHost>
+manifests:
+  certificates: true
+...

fluentd/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.10.1
 description: OpenStack-Helm Fluentd
 name: fluentd
-version: 0.1.2
+version: 0.1.3
 home: https://www.fluentd.org/
 sources:
   - https://github.com/fluent/fluentd

fluentd/templates/daemonset.yaml

@@ -140,6 +140,8 @@ spec:
               value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" | quote }}
             - name: ELASTICSEARCH_PORT
               value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: ELASTICSEARCH_SCHEME
+              value: {{ tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | quote }}
             - name: KAFKA_BROKER
               value: {{ $kafkaBrokerURI }}
 {{- if .Values.pod.env.fluentd.vars }}
@@ -194,6 +196,7 @@ spec:
               mountPath: /tmp/fluentd.sh
               subPath: fluentd.sh
               readOnly: true
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
 {{ if $mounts_fluentd.volumeMounts }}{{ toYaml $mounts_fluentd.volumeMounts | indent 12 }}{{- end }}
       volumes:
         - name: pod-tmp
@@ -220,5 +223,6 @@ spec:
           configMap:
             name: {{ printf "%s-%s" $envAll.Release.Name "fluentd-bin" | quote }}
             defaultMode: 0555
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{ if $mounts_fluentd.volumes }}{{ toYaml $mounts_fluentd.volumes | indent 8 }}{{- end }}
 {{- end }}

fluentd/values.yaml

@@ -120,6 +120,9 @@ endpoints:
       admin:
         username: admin
         password: changeme
+        secret:
+          tls:
+            internal: elasticsearch-tls-api
     hosts:
       data: elasticsearch-data
       default: elasticsearch-logging

fluentd/values_overrides/tls.yaml

@@ -0,0 +1,41 @@
+---
+conf:
+  fluentd:
+    conf:
+      output: |
+        <label @output>
+          <match **>
+            <buffer>
+              chunk_limit_size 512K
+              flush_interval 5s
+              flush_thread_count 8
+              queue_limit_length 32
+              retry_forever false
+              retry_max_interval 30
+            </buffer>
+            host "#{ENV['ELASTICSEARCH_HOST']}"
+            reload_connections false
+            reconnect_on_error true
+            reload_on_failure true
+            include_tag_key true
+            logstash_format true
+            password "#{ENV['ELASTICSEARCH_PASSWORD']}"
+            port "#{ENV['ELASTICSEARCH_PORT']}"
+            scheme "#{ENV['ELASTICSEARCH_SCHEME']}"
+            @type elasticsearch
+            user "#{ENV['ELASTICSEARCH_USERNAME']}"
+            ssl_verify true
+            ssl_version TLSv1_2
+            ca_file /etc/elasticsearch/certs/ca.crt
+          </match>
+        </label>
+endpoints:
+  elasticsearch:
+    scheme:
+      default: "https"
+    port:
+      http:
+        default: 443
+manifests:
+  certificates: true
+...

kibana/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v7.1.0
 description: OpenStack-Helm Kibana
 name: kibana
-version: 0.1.2
+version: 0.1.3
 home: https://www.elastic.co/products/kibana
 sources:
   - https://github.com/elastic/kibana

kibana/templates/bin/_flush_kibana_metadata.sh.tpl

@@ -15,5 +15,5 @@ limitations under the License.
 set -ex
 echo "Deleting index created for metadata"
 
-curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
   -XDELETE "${ELASTICSEARCH_ENDPOINT}/.kibana*"

kibana/templates/deployment.yaml

@@ -154,6 +154,7 @@ spec:
               mountPath: /usr/share/kibana/config/kibana.yml
               subPath: kibana.yml
               readOnly: true
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
       volumes:
         - name: pod-tmp
           emptyDir: {}
@@ -172,4 +173,5 @@ spec:
           secret:
             secretName: kibana-etc
             defaultMode: 0444
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{- end }}

kibana/templates/job-flush-kibana-metadata.yaml

@@ -75,7 +75,11 @@ spec:
             - name: KIBANA_ENDPOINT
               value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
             - name: ELASTICSEARCH_ENDPOINT
-              value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+              value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }}
+{{- if .Values.manifests.certificates }}
+            - name: CACERT_OPTION
+              value: "--cacert /etc/elasticsearch/certs/ca.crt"
+{{- end }}
           command:
             - /tmp/flush_kibana_metadata.sh
           volumeMounts:
@@ -87,6 +91,7 @@ spec:
               mountPath: /tmp/flush_kibana_metadata.sh
               subPath: flush_kibana_metadata.sh
               readOnly: false
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
       volumes:
         - name: pod-tmp
           emptyDir: {}
@@ -97,4 +102,5 @@ spec:
           configMap:
             name: kibana-bin
             defaultMode: 0755
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{- end }}

kibana/values.yaml

@@ -337,6 +337,9 @@ endpoints:
       admin:
         username: admin
         password: changeme
+        secret:
+          tls:
+            internal: elasticsearch-tls-api
     hosts:
       default: elasticsearch-logging
       public: elasticsearch

kibana/values_overrides/tls.yaml

@@ -0,0 +1,17 @@
+---
+conf:
+  kibana:
+    elasticsearch:
+      ssl:
+        certificateAuthorities: ["/etc/elasticsearch/certs/ca.crt"]
+        verificationMode: certificate
+endpoints:
+  elasticsearch:
+    scheme:
+      default: "https"
+    port:
+      http:
+        default: 443
+manifests:
+  certificates: true
+...

releasenotes/notes/elasticsearch.yaml

@@ -14,4 +14,5 @@ elasticsearch:
   - 0.2.2 Update the ES curator config to {}
   - 0.2.3 Add configurable backoffLimit to templates job
   - 0.2.4 Update helm-test script
+  - 0.2.5 Enable TLS with Kibana
 ...

releasenotes/notes/fluentd.yaml

@@ -3,4 +3,5 @@ fluentd:
   - 0.1.0 Initial Chart
   - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0"
   - 0.1.2 Add Configurable Readiness and Liveness Probes
+  - 0.1.3 Enable TLS path for output to Elasticsearch
 ...

releasenotes/notes/kibana.yaml

@@ -3,4 +3,5 @@ kibana:
   - 0.1.0 Initial Chart
   - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0"
   - 0.1.2 Drop usage of fsGroup inside container
+  - 0.1.3 Enable TLS with Elasticsearch
 ...