Enable TLS between Elasticsearch and Kibana

This change enables TLS between Elasticsearch and Kibana data path. Note that TLS terminates at apache-proxy container of the Elasticsearch-client pod, not directly to port 9200 of elasticsearch-client container. Since all data traffic goes through apache-proxy container, fluentd output to Elasticsearch are configured to have TLS enabled as well. In additon, other Elasticsearch pods that communicate with Elasticsearch-client endpoint are modified to provide the cacert option with curl. Change-Id: I3373c0c350b30c175be4a34d25a403b9caf74294
2021-04-08 12:34:35 -07:00 · 2021-04-08 12:34:35 -07:00 · 9a719e2a18
commit 9a719e2a18
parent a2c1eea8a9
30 changed files with 326 additions and 46 deletions
--- a/elasticsearch/Chart.yaml
+++ b/elasticsearch/Chart.yaml
@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v7.6.2
 description: OpenStack-Helm ElasticSearch
 name: elasticsearch
-version: 0.2.4
+version: 0.2.5
 home: https://www.elastic.co/
 sources:
  - https://github.com/elastic/elasticsearch
--- a/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl
+++ b/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl
@ -53,6 +53,8 @@ RGW_PROTO={{ $client.settings.protocol | default (tuple "ceph_object_store" "int
 CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST"
 if [ "$RGW_PROTO" = "http" ]; then
  CONNECTION_ARGS+=" --no-ssl"
+else
+  CONNECTION_ARGS+=" --no-check-certificate"
 fi

 USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY"
--- a/elasticsearch/templates/bin/_create_template.sh.tpl
+++ b/elasticsearch/templates/bin/_create_template.sh.tpl
@ -21,9 +21,9 @@ NUM_ERRORS=0
 {{ if not (empty $object) }}

 echo "creating {{$name}}"
-error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+error=$(curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
   -X{{ $object.method | default "PUT" | upper }} \
-   "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/{{ $object.endpoint }}" \
+   "${ELASTICSEARCH_ENDPOINT}/{{ $object.endpoint }}" \
   -H 'Content-Type: application/json' -d '{{ $object.body | toJson }}' | jq -r '.error')

 if [ $error == "null" ]; then
--- a/elasticsearch/templates/bin/_elasticsearch.sh.tpl
+++ b/elasticsearch/templates/bin/_elasticsearch.sh.tpl
@ -45,11 +45,11 @@ function stop () {
 function wait_to_join() {
  # delay 5 seconds before the first check
  sleep 5
-  joined=$(curl -s -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/_cat/nodes" | grep -w $NODE_NAME || true )
+  joined=$(curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/_cat/nodes" | grep -w $NODE_NAME || true )
  i=0
  while [ -z "$joined" ]; do
    sleep 5
-    joined=$(curl -s -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/_cat/nodes" | grep -w $NODE_NAME || true )
+    joined=$(curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/_cat/nodes" | grep -w $NODE_NAME || true )
    i=$((i+1))
    # Waiting for up to 60 minutes
    if [ $i -gt 720 ]; then
@ -62,7 +62,7 @@ function allocate_data_node () {
  echo "Node ${NODE_NAME} has started. Waiting to rejoin the cluster."
  wait_to_join
  echo "Re-enabling Replica Shard Allocation"
-  curl -s -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPUT -H 'Content-Type: application/json' \
+  curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPUT -H 'Content-Type: application/json' \
    "${ELASTICSEARCH_ENDPOINT}/_cluster/settings" -d "{
    \"persistent\": {
      \"cluster.routing.allocation.enable\": null
@ -102,7 +102,7 @@ function start_data_node () {
    # https://www.elastic.co/guide/en/elasticsearch/reference/7.x/restart-cluster.html#restart-cluster-rolling

    echo "Disabling Replica Shard Allocation"
-    curl -s -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPUT -H 'Content-Type: application/json' \
+    curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPUT -H 'Content-Type: application/json' \
      "${ELASTICSEARCH_ENDPOINT}/_cluster/settings" -d "{
      \"persistent\": {
        \"cluster.routing.allocation.enable\": \"primaries\"
@ -112,7 +112,7 @@ function start_data_node () {
    # If version < 7.6 use _flush/synced; otherwise use _flush
    # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-synced-flush-api.html#indices-synced-flush-api

-    version=$(curl -s -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/" | jq -r .version.number)
+    version=$(curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/" | jq -r .version.number)

    if [[ $version =~ "7.1" ]]; then
      action="_flush/synced"
@ -120,7 +120,7 @@ function start_data_node () {
      action="_flush"
    fi

-    curl -s -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPOST "${ELASTICSEARCH_ENDPOINT}/$action"
+    curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPOST "${ELASTICSEARCH_ENDPOINT}/$action"

    # TODO: Check the response of synced flush operations to make sure there are no failures.
    # Synced flush operations that fail due to pending indexing operations are listed in the response body,
--- a/elasticsearch/templates/bin/_helm-tests.sh.tpl
+++ b/elasticsearch/templates/bin/_helm-tests.sh.tpl
@ -16,7 +16,7 @@ limitations under the License.
 set -ex

 function create_test_index () {
-  index_result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  index_result=$(curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
  -XPUT "${ELASTICSEARCH_ENDPOINT}/test_index?pretty" -H 'Content-Type: application/json' -d'
  {
    "settings" : {
@ -38,13 +38,13 @@ function create_test_index () {

 {{ if .Values.conf.elasticsearch.snapshots.enabled }}
 function check_snapshot_repositories_verified () {
-  repositories=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  repositories=$(curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
                  "${ELASTICSEARCH_ENDPOINT}/_snapshot" | jq -r "keys | @sh" )

  repositories=$(echo $repositories | sed "s/'//g") # Strip single quotes from jq output

  for repository in $repositories; do
-    error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+    error=$(curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
            -XPOST "${ELASTICSEARCH_ENDPOINT}/_snapshot/${repository}/_verify" | jq -r '.error')

    if [ $error == "null" ]; then
@ -59,7 +59,7 @@ function check_snapshot_repositories_verified () {

 function remove_test_index () {
  echo "Deleting index created for service testing"
-  curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
  -XDELETE "${ELASTICSEARCH_ENDPOINT}/test_index"
 }

--- a/elasticsearch/templates/bin/_verify-repositories.sh.tpl
+++ b/elasticsearch/templates/bin/_verify-repositories.sh.tpl
@ -18,12 +18,12 @@ limitations under the License.
 set -ex

 function verify_snapshot_repository() {
-  curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-    -XPOST "${ELASTICSEARCH_HOST}/_snapshot/$1/_verify"
+  curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+    -XPOST "${ELASTICSEARCH_ENDPOINT}/_snapshot/$1/_verify"
 }

-repositories=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
-                "${ELASTICSEARCH_HOST}/_snapshot" | jq -r 'keys | @sh')
+repositories=$(curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+                "${ELASTICSEARCH_ENDPOINT}/_snapshot" | jq -r 'keys | @sh')

 repositories=$(echo $repositories | sed "s/'//g") # Strip single quotes from jq output

--- a/elasticsearch/templates/certificates.yaml
+++ b/elasticsearch/templates/certificates.yaml
@ -0,0 +1,17 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.certificates -}}
+{{  dict "envAll" . "service" "elasticsearch" "type" "internal" | include "helm-toolkit.manifests.certificates" }}
+{{- end -}}
--- a/elasticsearch/templates/cron-job-verify-repositories.yaml
+++ b/elasticsearch/templates/cron-job-verify-repositories.yaml
@ -70,8 +70,12 @@ spec:
                    secretKeyRef:
                      name: {{ $esUserSecret }}
                      key: ELASTICSEARCH_PASSWORD
-                - name: ELASTICSEARCH_HOST
-                  value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+                - name: ELASTICSEARCH_ENDPOINT
+                  value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }}
+{{- if .Values.manifests.certificates }}
+                - name: CACERT_OPTION
+                  value: "--cacert /etc/elasticsearch/certs/ca.crt"
+{{- end }}
              volumeMounts:
                - name: pod-tmp
                  mountPath: /tmp
@ -79,6 +83,7 @@ spec:
                  mountPath: /tmp/verify-repositories.sh
                  subPath: verify-repositories.sh
                  readOnly: true
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }}
          volumes:
            - name: pod-tmp
              emptyDir: {}
@ -86,4 +91,5 @@ spec:
              configMap:
                name: elasticsearch-bin
                defaultMode: 0555
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 12 }}
 {{- end }}
--- a/elasticsearch/templates/deployment-client.yaml
+++ b/elasticsearch/templates/deployment-client.yaml
@ -12,6 +12,20 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}

+{{- define "probeTemplate" }}
+{{- $probePort := tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $probeUser := .Values.endpoints.elasticsearch.auth.admin.username }}
+{{- $probePass := .Values.endpoints.elasticsearch.auth.admin.password }}
+{{- $authHeader := printf "%s:%s" $probeUser $probePass | b64enc }}
+httpGet:
+  path: /_cluster/health
+  scheme: {{ tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }}
+  port: {{ $probePort }}
+  httpHeaders:
+    - name: Authorization
+      value: Basic {{ $authHeader }}
+{{- end }}
+
 {{- if .Values.manifests.deployment_client }}
 {{- $envAll := . }}

@ -73,7 +87,7 @@ spec:
            - /tmp/apache.sh
            - start
          ports:
-            - name: http
+            - name: {{ tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
              containerPort: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
          readinessProbe:
            tcpSocket:
@ -112,6 +126,7 @@ spec:
              mountPath: /usr/local/apache2/conf/httpd.conf
              subPath: httpd.conf
              readOnly: true
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
        - name: elasticsearch-client
 {{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.client | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@ -126,8 +141,6 @@ spec:
                  - /tmp/elasticsearch.sh
                  - stop
          ports:
-            - name: http
-              containerPort: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
            - name: transport
              containerPort: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
          livenessProbe:
@ -135,10 +148,7 @@ spec:
              port: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
            initialDelaySeconds: 20
            periodSeconds: 10
-          readinessProbe:
-            httpGet:
-              path: /_cluster/health
-              port: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{ dict "envAll" . "component" "elasticsearch" "container" "elasticsearch-client" "type" "readiness" "probeTemplate" (include "probeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
          env:
            - name: NAMESPACE
              valueFrom:
@ -210,5 +220,6 @@ spec:
            defaultMode: 0444
        - name: storage
          emptyDir: {}
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
 {{- end }}
--- a/elasticsearch/templates/ingress-elasticsearch.yaml
+++ b/elasticsearch/templates/ingress-elasticsearch.yaml
@ -13,6 +13,12 @@ limitations under the License.
 */}}

 {{- if and .Values.manifests.ingress .Values.network.elasticsearch.ingress.public }}
-{{- $ingressOpts := dict "envAll" . "backendService" "elasticsearch" "backendServiceType" "elasticsearch" "backendPort" "http" -}}
+{{- $envAll := . -}}
+{{- $port := tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
+{{- $ingressOpts := dict "envAll" $envAll "backendService" "elasticsearch" "backendServiceType" "elasticsearch" "backendPort" $port -}}
+{{- $secretName := $envAll.Values.secrets.tls.elasticsearch.elasticsearch.internal -}}
+{{- if and .Values.manifests.certificates $secretName -}}
+{{- $_ := set $ingressOpts "certIssuer" .Values.endpoints.elasticsearch.host_fqdn_override.default.tls.issuerRef.name -}}
+{{- end -}}
 {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
 {{- end }}
--- a/elasticsearch/templates/job-elasticsearch-template.yaml
+++ b/elasticsearch/templates/job-elasticsearch-template.yaml
@ -50,10 +50,12 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_templates | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
 {{ dict "envAll" $envAll "application" "create_template" "container" "create_elasticsearch_template" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
          env:
-            - name: ELASTICSEARCH_HOST
-              value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
-            - name: ELASTICSEARCH_PORT
-              value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: ELASTICSEARCH_ENDPOINT
+              value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }}
+{{- if .Values.manifests.certificates }}
+            - name: CACERT_OPTION
+              value: "--cacert /etc/elasticsearch/certs/ca.crt"
+{{- end }}
            - name: ELASTICSEARCH_USERNAME
              valueFrom:
                secretKeyRef:
@ -73,6 +75,7 @@ spec:
              mountPath: /tmp/create_template.sh
              subPath: create_template.sh
              readOnly: true
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
 {{ if $mounts_elasticsearch_templates.volumeMounts }}{{ toYaml $mounts_elasticsearch_templates.volumeMounts | indent 12 }}{{ end }}
      volumes:
        - name: pod-tmp
@ -81,5 +84,6 @@ spec:
          configMap:
            name: elasticsearch-bin
            defaultMode: 0555
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{ if $mounts_elasticsearch_templates.volumes }}{{ toYaml $mounts_elasticsearch_templates.volumes | indent 8 }}{{ end }}
 {{- end }}
--- a/elasticsearch/templates/pod-helm-tests.yaml
+++ b/elasticsearch/templates/pod-helm-tests.yaml
@ -56,7 +56,11 @@ spec:
              name: {{ $esUserSecret }}
              key: ELASTICSEARCH_PASSWORD
        - name: ELASTICSEARCH_ENDPOINT
-          value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+          value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }}
+{{- if .Values.manifests.certificates }}
+        - name: CACERT_OPTION
+          value: "--cacert /etc/elasticsearch/certs/ca.crt"
+{{- end }}
      volumeMounts:
        - name: pod-tmp
          mountPath: /tmp
@ -64,6 +68,7 @@ spec:
          mountPath: /tmp/helm-tests.sh
          subPath: helm-tests.sh
          readOnly: true
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 8 }}
  volumes:
    - name: pod-tmp
      emptyDir: {}
@ -71,4 +76,5 @@ spec:
      configMap:
        name: elasticsearch-bin
        defaultMode: 0555
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 4 }}
 {{- end }}
--- a/elasticsearch/templates/secret-elasticsearch.yaml
+++ b/elasticsearch/templates/secret-elasticsearch.yaml
@ -19,7 +19,8 @@ limitations under the License.
 {{- $elasticsearch_user := .Values.endpoints.elasticsearch.auth.admin.username }}
 {{- $elasticsearch_password := .Values.endpoints.elasticsearch.auth.admin.password }}
 {{- $elasticsearch_host := tuple "elasticsearch" "internal" "http" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
-{{- $elasticsearch_uri := printf "http://%s:%s@%s" $elasticsearch_user $elasticsearch_password $elasticsearch_host }}
+{{- $elasticsearch_scheme := tuple "elasticsearch" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
+{{- $elasticsearch_uri := printf "%s://%s:%s@%s" $elasticsearch_scheme $elasticsearch_user $elasticsearch_password $elasticsearch_host }}
 ---
 apiVersion: v1
 kind: Secret
--- a/elasticsearch/templates/service-logging.yaml
+++ b/elasticsearch/templates/service-logging.yaml
@ -21,8 +21,9 @@ metadata:
  name: {{ tuple "elasticsearch" "default" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 spec:
  ports:
-  - name: http
+  - name: {{ tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
    port: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    targetPort: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
    {{- if .Values.network.elasticsearch.node_port.enabled }}
    nodePort: {{ .Values.network.elasticsearch.node_port.port }}
    {{- end }}
--- a/elasticsearch/templates/statefulset-data.yaml
+++ b/elasticsearch/templates/statefulset-data.yaml
@ -114,7 +114,11 @@ spec:
                  name: {{ $esUserSecret }}
                  key: ELASTICSEARCH_PASSWORD
            - name: ELASTICSEARCH_ENDPOINT
-              value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+              value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }}
+{{- if .Values.manifests.certificates }}
+            - name: CACERT_OPTION
+              value: "--cacert /etc/elasticsearch/certs/ca.crt"
+{{- end }}
            - name: NODE_MASTER
              value: "false"
            - name: NODE_INGEST
@ -158,6 +162,7 @@ spec:
              readOnly: true
            - name: storage
              mountPath: {{ .Values.conf.elasticsearch.config.path.data }}
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
 {{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
      volumes:
        - name: pod-tmp
@ -172,6 +177,7 @@ spec:
          secret:
            secretName: elasticsearch-etc
            defaultMode: 0444
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }}
 {{- if not .Values.storage.data.enabled }}
        - name: storage
--- a/elasticsearch/values.yaml
+++ b/elasticsearch/values.yaml
@ -289,6 +289,14 @@ pod:
        timeout: 600
      prometheus_elasticsearch_exporter:
        timeout: 600
+  probes:
+    elasticsearch:
+      elasticsearch-client:
+        readiness:
+          enabled: true
+          params:
+            initialDelaySeconds: 30
+            timeoutSeconds: 30
  mounts:
    elasticsearch:
      elasticsearch:
@ -418,6 +426,7 @@ secrets:
    elasticsearch:
      elasticsearch:
        public: elasticsearch-tls-public
+        internal: elasticsearch-tls-api

 jobs:
  curator:
@ -788,13 +797,6 @@ endpoints:
      public: elasticsearch
    host_fqdn_override:
      default: null
-      # NOTE(srwilkers): this chart supports TLS for fqdn over-ridden public
-      # endpoints using the following format:
-      # public:
-      #   host: null
-      #   tls:
-      #     crt: null
-      #     key: null
    path:
      default: null
    scheme:
@ -932,6 +934,7 @@ storage:
    #       - --region="default:backup"

 manifests:
+  certificates: false
  configmap_bin_curator: false
  configmap_bin_elasticsearch: true
  configmap_etc_curator: false
--- a/elasticsearch/values_overrides/tls.yaml
+++ b/elasticsearch/values_overrides/tls.yaml
@ -0,0 +1,138 @@
+---
+endpoints:
+  elasticsearch:
+    host_fqdn_override:
+      default:
+        tls:
+          secretName: elasticsearch-tls-api
+          issuerRef:
+            name: ca-issuer
+            kind: ClusterIssuer
+    scheme:
+      default: "https"
+    port:
+      http:
+        default: 443
+network:
+  elasticsearch:
+    ingress:
+      annotations:
+        nginx.ingress.kubernetes.io/backend-protocol: https
+conf:
+  httpd: |
+    ServerRoot "/usr/local/apache2"
+
+    Listen 443
+
+    LoadModule allowmethods_module modules/mod_allowmethods.so
+    LoadModule mpm_event_module modules/mod_mpm_event.so
+    LoadModule authn_file_module modules/mod_authn_file.so
+    LoadModule authn_core_module modules/mod_authn_core.so
+    LoadModule authz_host_module modules/mod_authz_host.so
+    LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
+    LoadModule authz_user_module modules/mod_authz_user.so
+    LoadModule authz_core_module modules/mod_authz_core.so
+    LoadModule access_compat_module modules/mod_access_compat.so
+    LoadModule auth_basic_module modules/mod_auth_basic.so
+    LoadModule ldap_module modules/mod_ldap.so
+    LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
+    LoadModule reqtimeout_module modules/mod_reqtimeout.so
+    LoadModule filter_module modules/mod_filter.so
+    LoadModule proxy_html_module modules/mod_proxy_html.so
+    LoadModule log_config_module modules/mod_log_config.so
+    LoadModule env_module modules/mod_env.so
+    LoadModule headers_module modules/mod_headers.so
+    LoadModule setenvif_module modules/mod_setenvif.so
+    LoadModule version_module modules/mod_version.so
+    LoadModule proxy_module modules/mod_proxy.so
+    LoadModule proxy_connect_module modules/mod_proxy_connect.so
+    LoadModule proxy_http_module modules/mod_proxy_http.so
+    LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
+    LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
+    LoadModule slotmem_plain_module modules/mod_slotmem_plain.so
+    LoadModule unixd_module modules/mod_unixd.so
+    LoadModule status_module modules/mod_status.so
+    LoadModule autoindex_module modules/mod_autoindex.so
+    LoadModule rewrite_module modules/mod_rewrite.so
+    LoadModule ssl_module modules/mod_ssl.so
+
+    <IfModule unixd_module>
+    User daemon
+    Group daemon
+    </IfModule>
+
+    <Directory />
+        AllowOverride none
+        Require all denied
+    </Directory>
+
+    <Files ".ht*">
+        Require all denied
+    </Files>
+
+    ErrorLog /dev/stderr
+
+    LogLevel warn
+
+    <IfModule log_config_module>
+        LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+        LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy
+        LogFormat "%h %l %u %t \"%r\" %>s %b" common
+
+        <IfModule logio_module>
+          LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
+        </IfModule>
+
+        SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
+        CustomLog /dev/stdout common
+        CustomLog /dev/stdout combined
+        CustomLog /dev/stdout proxy env=forwarded
+    </IfModule>
+
+    <Directory "/usr/local/apache2/cgi-bin">
+        AllowOverride None
+        Options None
+        Require all granted
+    </Directory>
+
+    <IfModule headers_module>
+        RequestHeader unset Proxy early
+    </IfModule>
+
+    <IfModule proxy_html_module>
+    Include conf/extra/proxy-html.conf
+    </IfModule>
+
+    <VirtualHost *:443>
+      <Location />
+          ProxyPass http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
+          ProxyPassReverse http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
+          AuthName "Elasticsearch"
+          AuthType Basic
+          AuthBasicProvider file ldap
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }}
+          AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }}
+          AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+          Require valid-user
+      </Location>
+
+      # Restrict access to the Elasticsearch Update By Query API Endpoint to prevent modification of indexed documents
+      <Location /*/_update_by_query*>
+          Require all denied
+      </Location>
+      # Restrict access to the Elasticsearch Delete By Query API Endpoint to prevent deletion of indexed documents
+      <Location /*/_delete_by_query*>
+          Require all denied
+      </Location>
+      SSLEngine On
+      SSLProxyEngine on
+      SSLCertificateFile      /etc/elasticsearch/certs/tls.crt
+      SSLCertificateKeyFile   /etc/elasticsearch/certs/tls.key
+      SSLProtocol             all -SSLv3 -TLSv1 -TLSv1.1
+      SSLCipherSuite          ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
+      SSLHonorCipherOrder     on
+    </VirtualHost>
+manifests:
+  certificates: true
+...
--- a/fluentd/Chart.yaml
+++ b/fluentd/Chart.yaml
@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.10.1
 description: OpenStack-Helm Fluentd
 name: fluentd
-version: 0.1.2
+version: 0.1.3
 home: https://www.fluentd.org/
 sources:
  - https://github.com/fluent/fluentd
--- a/fluentd/templates/daemonset.yaml
+++ b/fluentd/templates/daemonset.yaml
@ -140,6 +140,8 @@ spec:
              value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" | quote }}
            - name: ELASTICSEARCH_PORT
              value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: ELASTICSEARCH_SCHEME
+              value: {{ tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | quote }}
            - name: KAFKA_BROKER
              value: {{ $kafkaBrokerURI }}
 {{- if .Values.pod.env.fluentd.vars }}
@ -194,6 +196,7 @@ spec:
              mountPath: /tmp/fluentd.sh
              subPath: fluentd.sh
              readOnly: true
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
 {{ if $mounts_fluentd.volumeMounts }}{{ toYaml $mounts_fluentd.volumeMounts | indent 12 }}{{- end }}
      volumes:
        - name: pod-tmp
@ -220,5 +223,6 @@ spec:
          configMap:
            name: {{ printf "%s-%s" $envAll.Release.Name "fluentd-bin" | quote }}
            defaultMode: 0555
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{ if $mounts_fluentd.volumes }}{{ toYaml $mounts_fluentd.volumes | indent 8 }}{{- end }}
 {{- end }}
--- a/fluentd/values.yaml
+++ b/fluentd/values.yaml
@ -120,6 +120,9 @@ endpoints:
      admin:
        username: admin
        password: changeme
+        secret:
+          tls:
+            internal: elasticsearch-tls-api
    hosts:
      data: elasticsearch-data
      default: elasticsearch-logging
--- a/fluentd/values_overrides/tls.yaml
+++ b/fluentd/values_overrides/tls.yaml
@ -0,0 +1,41 @@
+---
+conf:
+  fluentd:
+    conf:
+      output: |
+        <label @output>
+          <match **>
+            <buffer>
+              chunk_limit_size 512K
+              flush_interval 5s
+              flush_thread_count 8
+              queue_limit_length 32
+              retry_forever false
+              retry_max_interval 30
+            </buffer>
+            host "#{ENV['ELASTICSEARCH_HOST']}"
+            reload_connections false
+            reconnect_on_error true
+            reload_on_failure true
+            include_tag_key true
+            logstash_format true
+            password "#{ENV['ELASTICSEARCH_PASSWORD']}"
+            port "#{ENV['ELASTICSEARCH_PORT']}"
+            scheme "#{ENV['ELASTICSEARCH_SCHEME']}"
+            @type elasticsearch
+            user "#{ENV['ELASTICSEARCH_USERNAME']}"
+            ssl_verify true
+            ssl_version TLSv1_2
+            ca_file /etc/elasticsearch/certs/ca.crt
+          </match>
+        </label>
+endpoints:
+  elasticsearch:
+    scheme:
+      default: "https"
+    port:
+      http:
+        default: 443
+manifests:
+  certificates: true
+...
--- a/kibana/Chart.yaml
+++ b/kibana/Chart.yaml
@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v7.1.0
 description: OpenStack-Helm Kibana
 name: kibana
-version: 0.1.2
+version: 0.1.3
 home: https://www.elastic.co/products/kibana
 sources:
  - https://github.com/elastic/kibana
--- a/kibana/templates/bin/_flush_kibana_metadata.sh.tpl
+++ b/kibana/templates/bin/_flush_kibana_metadata.sh.tpl
@ -15,5 +15,5 @@ limitations under the License.
 set -ex
 echo "Deleting index created for metadata"

-curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
  -XDELETE "${ELASTICSEARCH_ENDPOINT}/.kibana*"
--- a/kibana/templates/deployment.yaml
+++ b/kibana/templates/deployment.yaml
@ -154,6 +154,7 @@ spec:
              mountPath: /usr/share/kibana/config/kibana.yml
              subPath: kibana.yml
              readOnly: true
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
      volumes:
        - name: pod-tmp
          emptyDir: {}
@ -172,4 +173,5 @@ spec:
          secret:
            secretName: kibana-etc
            defaultMode: 0444
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{- end }}
--- a/kibana/templates/job-flush-kibana-metadata.yaml
+++ b/kibana/templates/job-flush-kibana-metadata.yaml
@ -75,7 +75,11 @@ spec:
            - name: KIBANA_ENDPOINT
              value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
            - name: ELASTICSEARCH_ENDPOINT
-              value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+              value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }}
+{{- if .Values.manifests.certificates }}
+            - name: CACERT_OPTION
+              value: "--cacert /etc/elasticsearch/certs/ca.crt"
+{{- end }}
          command:
            - /tmp/flush_kibana_metadata.sh
          volumeMounts:
@ -87,6 +91,7 @@ spec:
              mountPath: /tmp/flush_kibana_metadata.sh
              subPath: flush_kibana_metadata.sh
              readOnly: false
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
      volumes:
        - name: pod-tmp
          emptyDir: {}
@ -97,4 +102,5 @@ spec:
          configMap:
            name: kibana-bin
            defaultMode: 0755
+{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{- end }}
--- a/kibana/values.yaml
+++ b/kibana/values.yaml
@ -337,6 +337,9 @@ endpoints:
      admin:
        username: admin
        password: changeme
+        secret:
+          tls:
+            internal: elasticsearch-tls-api
    hosts:
      default: elasticsearch-logging
      public: elasticsearch
--- a/kibana/values_overrides/tls.yaml
+++ b/kibana/values_overrides/tls.yaml
@ -0,0 +1,17 @@
+---
+conf:
+  kibana:
+    elasticsearch:
+      ssl:
+        certificateAuthorities: ["/etc/elasticsearch/certs/ca.crt"]
+        verificationMode: certificate
+endpoints:
+  elasticsearch:
+    scheme:
+      default: "https"
+    port:
+      http:
+        default: 443
+manifests:
+  certificates: true
+...
--- a/releasenotes/notes/elasticsearch.yaml
+++ b/releasenotes/notes/elasticsearch.yaml
@ -14,4 +14,5 @@ elasticsearch:
  - 0.2.2 Update the ES curator config to {}
  - 0.2.3 Add configurable backoffLimit to templates job
  - 0.2.4 Update helm-test script
+  - 0.2.5 Enable TLS with Kibana
 ...
--- a/releasenotes/notes/fluentd.yaml
+++ b/releasenotes/notes/fluentd.yaml
@ -3,4 +3,5 @@ fluentd:
  - 0.1.0 Initial Chart
  - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0"
  - 0.1.2 Add Configurable Readiness and Liveness Probes
+  - 0.1.3 Enable TLS path for output to Elasticsearch
 ...
--- a/releasenotes/notes/kibana.yaml
+++ b/releasenotes/notes/kibana.yaml
@ -3,4 +3,5 @@ kibana:
  - 0.1.0 Initial Chart
  - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0"
  - 0.1.2 Drop usage of fsGroup inside container
+  - 0.1.3 Enable TLS with Elasticsearch
 ...