Run elasticsearch behind apache

Run elasticsearch behind apache as a reverse proxy to supply basic
auth for elasticsearch, as xpack requires a suscription to support
security for elasticsearch

Change-Id: I72d06ed9cd2179ead86ddc67db33c68a1e40c437

elasticsearch/templates/_kibana.sh.tpl

@@ -0,0 +1,32 @@
+#!/bin/bash
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+COMMAND="${@:-start}"
+
+function start () {
+  exec kibana \
+    --elasticsearch.url="${ELASTICSEARCH_URL}" \
+    --elasticsearch.username="${ELASTICSEARCH_USERNAME}" \
+    --elasticsearch.password="{$ELASTICSEARCH_PASSWORD}"
+}
+
+function stop () {
+  kill -TERM 1
+}
+
+$COMMAND

elasticsearch/templates/bin/_apache.sh.tpl

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ev
+
+COMMAND="${@:-start}"
+
+function start () {
+
+  if [ -f /etc/apache2/envvars ]; then
+     # Loading Apache2 ENV variables
+     source /etc/httpd/apache2/envvars
+  fi
+  # Apache gets grumpy about PID files pre-existing
+  rm -f /etc/httpd/logs/httpd.pid
+
+  if [ -f {{ .Values.conf.apache.htpasswd }} ]; then
+    htpasswd -b {{ .Values.conf.apache.htpasswd }} $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD
+  else
+    htpasswd -cb {{ .Values.conf.apache.htpasswd }} $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD
+  fi
+
+  #Launch Apache on Foreground
+  exec httpd -DFOREGROUND
+}
+
+function stop () {
+  apachectl -k graceful-stop
+}
+
+$COMMAND

elasticsearch/templates/bin/_helm-tests.sh.tpl

@@ -19,7 +19,8 @@ limitations under the License.
 set -ex
 
 function create_index () {
-  index_result=$(curl -XPUT "${ELASTICSEARCH_ENDPOINT}/test_index?pretty" -H 'Content-Type: application/json' -d'
+  index_result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  -XPUT "${ELASTICSEARCH_ENDPOINT}/test_index?pretty" -H 'Content-Type: application/json' -d'
   {
     "settings" : {
       "index" : {
@@ -39,7 +40,8 @@ function create_index () {
 }
 
 function insert_test_data () {
-  insert_result=$(curl -XPUT "${ELASTICSEARCH_ENDPOINT}/sample_index/sample_type/123/_create?pretty" -H 'Content-Type: application/json' -d'
+  insert_result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  -XPUT "${ELASTICSEARCH_ENDPOINT}/sample_index/sample_type/123/_create?pretty" -H 'Content-Type: application/json' -d'
   {
       "name" : "Elasticsearch",
       "message" : "Test data text entry"
@@ -56,7 +58,8 @@ function insert_test_data () {
 
 
 function check_hits () {
-  total_hits=$(curl -XGET "${ELASTICSEARCH_ENDPOINT}/_search?pretty" -H 'Content-Type: application/json' -d'
+  total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  "${ELASTICSEARCH_ENDPOINT}/_search?pretty" -H 'Content-Type: application/json' -d'
   {
     "query" : {
       "bool": {

elasticsearch/templates/bin/_register-repository.sh.tpl

@@ -17,7 +17,9 @@ limitations under the License.
 
 set -ex
 
-exec curl -X PUT "${ELASTICSEARCH_ENDPOINT}/_snapshot/${REPO_NAME}" -H 'Content-Type: application/json' -d'
+exec curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+  "${ELASTICSEARCH_ENDPOINT}/_snapshot/${REPO_NAME}" \
+  -H 'Content-Type: application/json' -d'
   {
     "type": "'"$REPO_TYPE"'",
     "settings": {

elasticsearch/templates/configmap-bin.yaml

@@ -22,6 +22,8 @@ kind: ConfigMap
 metadata:
   name: elasticsearch-bin
 data:
+  apache.sh: |
+{{ tuple "bin/_apache.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   elasticsearch.sh: |
 {{ tuple "bin/_elasticsearch.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   helm-tests.sh: |

elasticsearch/templates/configmap-etc.yaml

@@ -26,6 +26,10 @@ kind: ConfigMap
 metadata:
   name: elasticsearch-etc
 data:
+  httpd.conf: |+
+{{- tuple .Values.conf.apache.httpd "etc/_httpd.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }}
+  elasticsearch-host.conf: |+
+{{- tuple .Values.conf.apache.host "etc/_elasticsearch-host.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }}
   elasticsearch.yml: |+
 {{ toYaml .Values.conf.elasticsearch.config | indent 4 }}
   log4j2.properties: |+

elasticsearch/templates/deployment-client.yaml

@@ -16,6 +16,7 @@ limitations under the License.
 
 {{- if .Values.manifests.deployment_client }}
 {{- $envAll := . }}
+{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 {{- if .Values.images.local_registry.active -}}
 {{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.elasticsearch_client .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
@@ -108,6 +109,43 @@ spec:
               mountPath: {{ .Values.conf.elasticsearch.repository.location }}
 {{ end }}
       containers:
+        - name: apache-proxy
+{{ tuple $envAll "apache_proxy" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.apache_proxy | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/apache.sh
+            - start
+          ports:
+            - name: http
+              containerPort: 80
+          env:
+            - name: ELASTICSEARCH_PORT
+              value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: ELASTICSEARCH_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_USERNAME
+            - name: ELASTICSEARCH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_PASSWORD
+          volumeMounts:
+            - name: elasticsearch-bin
+              mountPath: /tmp/apache.sh
+              subPath: apache.sh
+              readOnly: true
+            - name: elasticsearch-etc
+              mountPath: /usr/local/apache2/conf/httpd.conf
+              subPath: httpd.conf
+              readOnly: true
+            - name: pod-etc-apache
+              mountPath: /usr/local/apache2/conf/sites-enabled
+            - name: elasticsearch-etc
+              mountPath: /usr/local/apache2/conf/sites-enabled/elasticsearch-host.conf
+              subPath: elasticsearch-host.conf
+              readOnly: true
         - name: elasticsearch-client
           securityContext:
             privileged: true
@@ -184,6 +222,8 @@ spec:
             {{ end }}
 {{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }}
       volumes:
+        - name: pod-etc-apache
+          emptyDir: {}
         - name: elasticsearch-logs
           emptyDir: {}
         - name: elasticsearch-bin

elasticsearch/templates/etc/_elasticsearch-host.conf.tpl

@@ -0,0 +1,28 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+<VirtualHost *:80>
+  <Location />
+      ProxyPass http://localhost:${ELASTICSEARCH_PORT}/
+      ProxyPassReverse http://localhost:${ELASTICSEARCH_PORT}/
+  </Location>
+  <Proxy *>
+      AuthType Basic
+      AuthName "Authentication Required"
+      AuthUserFile {{.Values.conf.apache.htpasswd | quote}}
+      Require valid-user
+  </Proxy>
+</VirtualHost>

elasticsearch/templates/etc/_httpd.conf.tpl

@@ -0,0 +1,186 @@
+#
+# This is the main Apache HTTP server configuration file.  It contains the
+# configuration directives that give the server its instructions.
+# See <URL:http://httpd.apache.org/docs/2.4/> for detailed information.
+# In particular, see
+# <URL:http://httpd.apache.org/docs/2.4/mod/directives.html>
+# for a discussion of each configuration directive.
+#
+# Do NOT simply read the instructions in here without understanding
+# what they do.  They're here only as hints or reminders.  If you are unsure
+# consult the online docs. You have been warned.
+#
+# Configuration and logfile names: If the filenames you specify for many
+# of the server's control files begin with "/" (or "drive:/" for Win32), the
+# server will use that explicit path.  If the filenames do *not* begin
+# with "/", the value of ServerRoot is prepended -- so "logs/access_log"
+# with ServerRoot set to "/usr/local/apache2" will be interpreted by the
+# server as "/usr/local/apache2/logs/access_log", whereas "/logs/access_log"
+# will be interpreted as '/logs/access_log'.
+
+ServerRoot "/usr/local/apache2"
+
+#
+# Listen: Allows you to bind Apache to specific IP addresses and/or
+# ports, instead of the default. See also the <VirtualHost>
+# directive.
+#
+# Change this to Listen on specific IP addresses as shown below to
+# prevent Apache from glomming onto all bound IP addresses.
+#
+#Listen 12.34.56.78:80
+Listen 80
+
+#
+# Dynamic Shared Object (DSO) Support
+#
+# To be able to use the functionality of a module which was built as a DSO you
+# have to place corresponding `LoadModule' lines at this location so the
+# directives contained in it are actually available _before_ they are used.
+# Statically compiled modules (those listed by `httpd -l') do not need
+# to be loaded here.
+#
+# Example:
+# LoadModule foo_module modules/mod_foo.so
+#
+LoadModule authn_file_module modules/mod_authn_file.so
+LoadModule authn_core_module modules/mod_authn_core.so
+LoadModule authz_host_module modules/mod_authz_host.so
+LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
+LoadModule authz_user_module modules/mod_authz_user.so
+LoadModule authz_core_module modules/mod_authz_core.so
+LoadModule access_compat_module modules/mod_access_compat.so
+LoadModule auth_basic_module modules/mod_auth_basic.so
+LoadModule reqtimeout_module modules/mod_reqtimeout.so
+LoadModule filter_module modules/mod_filter.so
+LoadModule proxy_html_module modules/mod_proxy_html.so
+LoadModule log_config_module modules/mod_log_config.so
+LoadModule env_module modules/mod_env.so
+LoadModule headers_module modules/mod_headers.so
+LoadModule setenvif_module modules/mod_setenvif.so
+LoadModule version_module modules/mod_version.so
+LoadModule proxy_module modules/mod_proxy.so
+LoadModule proxy_connect_module modules/mod_proxy_connect.so
+LoadModule proxy_http_module modules/mod_proxy_http.so
+LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
+LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
+LoadModule slotmem_plain_module modules/mod_slotmem_plain.so
+LoadModule unixd_module modules/mod_unixd.so
+LoadModule status_module modules/mod_status.so
+LoadModule autoindex_module modules/mod_autoindex.so
+
+<IfModule unixd_module>
+#
+# If you wish httpd to run as a different user or group, you must run
+# httpd as root initially and it will switch.
+#
+# User/Group: The name (or #number) of the user/group to run httpd as.
+# It is usually good practice to create a dedicated user and group for
+# running httpd, as with most system services.
+#
+User daemon
+Group daemon
+
+</IfModule>
+
+# 'Main' server configuration
+#
+# The directives in this section set up the values used by the 'main'
+# server, which responds to any requests that aren't handled by a
+# <VirtualHost> definition.  These values also provide defaults for
+# any <VirtualHost> containers you may define later in the file.
+#
+# All of these directives may appear inside <VirtualHost> containers,
+# in which case these default settings will be overridden for the
+# virtual host being defined.
+#
+
+#
+# Deny access to the entirety of your server's filesystem. You must
+# explicitly permit access to web content directories in other
+# <Directory> blocks below.
+#
+<Directory />
+    AllowOverride none
+    Require all denied
+</Directory>
+
+#
+# The following lines prevent .htaccess and .htpasswd files from being
+# viewed by Web clients.
+#
+<Files ".ht*">
+    Require all denied
+</Files>
+
+#
+# ErrorLog: The location of the error log file.
+# If you do not specify an ErrorLog directive within a <VirtualHost>
+# container, error messages relating to that virtual host will be
+# logged here.  If you *do* define an error logfile for a <VirtualHost>
+# container, that host's errors will be logged there and not here.
+#
+ErrorLog /dev/stderr
+
+#
+# LogLevel: Control the number of messages logged to the error_log.
+# Possible values include: debug, info, notice, warn, error, crit,
+# alert, emerg.
+#
+LogLevel warn
+
+<IfModule log_config_module>
+    #
+    # The following directives define some format nicknames for use with
+    # a CustomLog directive (see below).
+    #
+    LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+    LogFormat "%h %l %u %t \"%r\" %>s %b" common
+
+    <IfModule logio_module>
+      # You need to enable mod_logio.c to use %I and %O
+      LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
+    </IfModule>
+
+    #
+    # The location and format of the access logfile (Common Logfile Format).
+    # If you do not define any access logfiles within a <VirtualHost>
+    # container, they will be logged here.  Contrariwise, if you *do*
+    # define per-<VirtualHost> access logfiles, transactions will be
+    # logged therein and *not* in this file.
+    #
+    CustomLog /dev/stdout common
+
+    #
+    # If you prefer a logfile with access, agent, and referer information
+    # (Combined Logfile Format) you can use the following directive.
+    #
+    CustomLog /dev/stdout combined
+</IfModule>
+
+#
+# "/usr/local/apache2/cgi-bin" should be changed to whatever your ScriptAliased
+# CGI directory exists, if you have that configured.
+#
+<Directory "/usr/local/apache2/cgi-bin">
+    AllowOverride None
+    Options None
+    Require all granted
+</Directory>
+
+<IfModule headers_module>
+    #
+    # Avoid passing HTTP_PROXY environment to CGI's on this or any proxied
+    # backend servers which have lingering "httpoxy" defects.
+    # 'Proxy' request header is undefined by the IETF, not listed by IANA
+    #
+    RequestHeader unset Proxy early
+</IfModule>
+
+# Virtual hosts
+Include conf/sites-enabled/*.conf
+
+# Configure mod_proxy_html to understand HTML4/XHTML1
+<IfModule proxy_html_module>
+Include conf/extra/proxy-html.conf
+</IfModule>

elasticsearch/templates/job-register-snapshot-repository.yaml

@@ -17,6 +17,7 @@ limitations under the License.
 {{- if .Values.manifests.job_snapshot_repository }}
 {{- if .Values.conf.elasticsearch.repository.enabled }}
 {{- $envAll := . }}
+{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 {{- $_ := set .Values "pod_dependency" .Values.dependencies.snapshot_repository -}}
 
 {{- $serviceAccountName := "elasticsearch-register-snapshot-repository" }}
@@ -43,8 +44,18 @@ spec:
 {{ tuple $envAll "snapshot_repository" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.snapshot_repository | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           env:
+            - name: ELASTICSEARCH_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_USERNAME
+            - name: ELASTICSEARCH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_PASSWORD
             - name: ELASTICSEARCH_ENDPOINT
-              value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+              value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
             - name: REPO_NAME
               value: {{ .Values.conf.elasticsearch.repository.name | quote }}
             - name: REPO_TYPE

elasticsearch/templates/pod-helm-tests.yaml

@@ -16,6 +16,7 @@ limitations under the License.
 
 {{- if .Values.manifests.helm_tests }}
 {{- $envAll := . }}
+{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 ---
 apiVersion: v1
 kind: Pod
@@ -32,8 +33,18 @@ spec:
       command:
         - /tmp/helm-tests.sh
       env:
+        - name: ELASTICSEARCH_USERNAME
+          valueFrom:
+            secretKeyRef:
+              name: {{ $esUserSecret }}
+              key: ELASTICSEARCH_USERNAME
+        - name: ELASTICSEARCH_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ $esUserSecret }}
+              key: ELASTICSEARCH_PASSWORD
         - name: ELASTICSEARCH_ENDPOINT
-          value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+          value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
       volumeMounts:
         - name: elasticsearch-bin
           mountPath: /tmp/helm-tests.sh

elasticsearch/templates/secret-admin-creds.yaml

@@ -0,0 +1,29 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_admin }}
+{{- $envAll := . }}
+{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+  ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }}
+  ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }}
+{{- end }}

elasticsearch/templates/service-logging.yaml

@@ -24,7 +24,7 @@ metadata:
 spec:
   ports:
   - name: http
-    port: {{ .Values.network.client.port }}
+    port: 80
     {{- if .Values.network.client.node_port.enabled }}
     nodePort: {{ .Values.network.client.node_port.port }}
     {{- end }}

elasticsearch/values.yaml

@@ -18,6 +18,7 @@
 
 images:
   tags:
+    apache_proxy: docker.io/httpd:2.4
     memory_init: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
     curator:  docker.io/bobrik/curator:5.2.0
     elasticsearch: docker.io/elasticsearch:5.4.2
@@ -144,7 +145,14 @@ pod:
           memory: "1024Mi"
           cpu: "2000m"
 
+secrets:
+  elasticsearch:
+    user: elasticsearch-admin-creds
+
 conf:
+  apache:
+    htpasswd: /usr/local/apache2/conf/.htpasswd
+  httpd:
   init:
     max_map_count: 262144
   curator:
@@ -263,6 +271,10 @@ endpoints:
   elasticsearch:
     name: elasticsearch
     namespace: null
+    auth:
+      admin:
+        username: admin
+        password: changeme
     hosts:
       data: elasticsearch-data
       default: elasticsearch-logging
@@ -277,6 +289,8 @@ endpoints:
     port:
       client:
         default: 9200
+      http:
+        default: 80
       discovery:
         default: 9300
 
@@ -326,6 +340,7 @@ manifests:
   job_snapshot_repository: true
   helm_tests: true
   pvc_snapshots: true
+  secret_admin: true
   service_data: true
   service_discovery: true
   service_logging: true

fluent-logging/templates/bin/_helm-tests.sh.tpl

@@ -21,7 +21,8 @@ set -ex
 # Tests whether fluentd has successfully indexed data into Elasticsearch under
 # the logstash-* index via the fluent-elasticsearch plugin
 function check_logstash_index () {
-  total_hits=$(curl -XGET "${ELASTICSEARCH_ENDPOINT}/logstash-*/fluentd/_search?pretty" -H 'Content-Type: application/json' \
+  total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+              -XGET "${ELASTICSEARCH_ENDPOINT}/logstash-*/fluentd/_search?pretty" -H 'Content-Type: application/json' \
               | python -c "import sys, json; print json.load(sys.stdin)['hits']['total']")
   if [ "$total_hits" -gt 0 ]; then
      echo "PASS: Successful hits on logstash-* index, provided by fluentd!"
@@ -34,7 +35,8 @@ function check_logstash_index () {
 # Tests whether fluentd has successfully tagged data with the kube.*
 # prefix via the fluent-kubernetes plugin
 function check_kubernetes_tag () {
-  total_hits=$(curl -XGET "${ELASTICSEARCH_ENDPOINT}/logstash-*/fluentd/_search?q=tag:kube.*" -H 'Content-Type: application/json' \
+  total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
+              -XGET "${ELASTICSEARCH_ENDPOINT}/logstash-*/fluentd/_search?q=tag:kube.*" -H 'Content-Type: application/json' \
               | python -c "import sys, json; print json.load(sys.stdin)['hits']['total']")
   if [ "$total_hits" -gt 0 ]; then
      echo "PASS: Successful hits on logstash-* index, provided by fluentd!"

fluent-logging/templates/deployment-fluentd.yaml

@@ -16,6 +16,7 @@ limitations under the License.
 
 {{- if .Values.manifests.deployment_fluentd }}
 {{- $envAll := . }}
+{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 {{- if .Values.images.local_registry.active -}}
 {{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.fluentd .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
@@ -117,6 +118,16 @@ spec:
               value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
             - name: ELASTICSEARCH_PORT
               value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: ELASTICSEARCH_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_USERNAME
+            - name: ELASTICSEARCH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_PASSWORD
           volumeMounts:
             - name: pod-etc-fluentd
               mountPath: /etc/td-agent

fluent-logging/templates/pod-helm-tests.yaml

@@ -16,6 +16,7 @@ limitations under the License.
 
 {{- if .Values.manifests.helm_tests }}
 {{- $envAll := . }}
+{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 ---
 apiVersion: v1
 kind: Pod
@@ -31,6 +32,16 @@ spec:
       command:
       - /tmp/helm-tests.sh
       env:
+      - name: ELASTICSEARCH_USERNAME
+        valueFrom:
+          secretKeyRef:
+            name: {{ $esUserSecret }}
+            key: ELASTICSEARCH_USERNAME
+      - name: ELASTICSEARCH_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: {{ $esUserSecret }}
+            key: ELASTICSEARCH_PASSWORD
       - name: ELASTICSEARCH_ENDPOINT
         value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
       volumeMounts:

fluent-logging/templates/secret-elasticsearch-creds.yaml

@@ -0,0 +1,29 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_elasticsearch }}
+{{- $envAll := . }}
+{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+  ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }}
+  ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }}
+{{- end }}

fluent-logging/values.yaml

@@ -40,6 +40,10 @@ images:
       - dep_check
       - image_repo_sync
 
+secrets:
+  elasticsearch:
+    user: fluentd-elasticsearch-user
+
 dependencies:
   image_repo_sync:
     services:
@@ -122,6 +126,8 @@ conf:
     - elasticsearch:
         header: match
         type: elasticsearch
+        user: "#{ENV['ELASTICSEARCH_USERNAME']}"
+        password: "#{ENV['ELASTICSEARCH_PASSWORD']}"
         expression: "**"
         include_tag_key: true
         host: "#{ENV['ELASTICSEARCH_HOST']}"
@@ -134,12 +140,15 @@ conf:
         disable_retry_limit: ""
         num_threads: 8
 
-
 endpoints:
   cluster_domain_suffix: cluster.local
   elasticsearch:
     namespace: null
     name: elasticsearch
+    auth:
+      admin:
+        username: admin
+        password: changeme
     hosts:
       data: elasticsearch-data
       default: elasticsearch-logging
@@ -153,7 +162,7 @@ endpoints:
       default: http
     port:
       client:
-        default: 9200
+        default: 80
       discovery:
         default: 9300
   kafka:
@@ -251,7 +260,6 @@ pod:
         limits:
           memory: '1024Mi'
           cpu: '2000m'
-
   mounts:
     fluentd:
       fluentd:
@@ -267,4 +275,5 @@ manifests:
   daemonset_fluentbit: true
   job_image_repo_sync: true
   helm_tests: true
+  secret_elasticsearch: true
   service_fluentd: true

kibana/templates/bin/_kibana.sh.tpl

@@ -19,7 +19,10 @@ set -ex
 COMMAND="${@:-start}"
 
 function start () {
-  exec kibana --elasticsearch.url="${ELASTICSEARCH_URL}"
+  exec kibana \
+    --elasticsearch.url="$ELASTICSEARCH_URL" \
+    --elasticsearch.username="$ELASTICSEARCH_USERNAME" \
+    --elasticsearch.password="$ELASTICSEARCH_PASSWORD"
 }
 
 function stop () {

kibana/templates/deployment.yaml

@@ -16,6 +16,7 @@ limitations under the License.
 
 {{- if .Values.manifests.deployment }}
 {{- $envAll := . }}
+{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 {{- if .Values.images.local_registry.active -}}
 {{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.kibana .Values.conditional_dependencies.local_image_registry) -}}
 {{- else -}}
@@ -60,6 +61,16 @@ spec:
           env:
             - name: ELASTICSEARCH_URL
               value: {{ tuple "elasticsearch" "default" "client" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
+            - name: ELASTICSEARCH_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_USERNAME
+            - name: ELASTICSEARCH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_PASSWORD
           volumeMounts:
             - name: kibana-bin
               mountPath: /tmp/kibana.sh

kibana/templates/secret-elasticsearch-creds.yaml

@@ -0,0 +1,29 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_elasticsearch }}
+{{- $envAll := . }}
+{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+  ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }}
+  ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }}
+{{- end }}

kibana/values.yaml

@@ -64,6 +64,10 @@ pod:
           memory: "1024Mi"
           cpu: "2000m"
 
+secrets:
+  elasticsearch:
+    user: kibana-elasticsearch-user
+
 dependencies:
   kibana:
     services:
@@ -124,6 +128,10 @@ endpoints:
   elasticsearch:
     name: elasticsearch
     namespace: null
+    auth:
+      admin:
+        username: admin
+        password: changeme
     hosts:
       default: elasticsearch-logging
       public: elasticsearch
@@ -135,7 +143,7 @@ endpoints:
       default: http
     port:
       client:
-        default: 9200
+        default: 80
   kibana:
     name: kibana
     namespace: null
@@ -168,5 +176,6 @@ manifests:
   deployment: true
   ingress_kibana: true
   job_image_repo_sync: true
+  secret_elasticsearch: true
   service: true
   service_ingress_kibana: true