Add ldap support in elasticsearch/kibana apache proxies

This adds required configuration for enabling LDAP through the apache proxy in the elasticsearch and kibana charts by default Change-Id: Iaff8f328ff50944ddad94ec86b1134ca73750176
2018-04-20 14:54:05 -05:00 · 2018-04-20 14:54:05 -05:00 · 3c692abd6e
commit 3c692abd6e
parent 85208fe98a
20 changed files with 292 additions and 19 deletions
--- a/elasticsearch/templates/bin/_apache.sh.tpl
+++ b/elasticsearch/templates/bin/_apache.sh.tpl
@ -29,10 +29,10 @@ function start () {
  # Apache gets grumpy about PID files pre-existing
  rm -f /etc/httpd/logs/httpd.pid

-  if [ -f {{ .Values.conf.apache.htpasswd }} ]; then
-    htpasswd -b {{ .Values.conf.apache.htpasswd }} $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD
+  if [ -f /usr/local/apache2/conf/.htpasswd ]; then
+    htpasswd -b /usr/local/apache2/conf/.htpasswd $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD
  else
-    htpasswd -cb {{ .Values.conf.apache.htpasswd }} $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD
+    htpasswd -cb /usr/local/apache2/conf/.htpasswd $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD
  fi

  #Launch Apache on Foreground
--- a/elasticsearch/templates/deployment-client.yaml
+++ b/elasticsearch/templates/deployment-client.yaml
@ -129,6 +129,18 @@ spec:
                secretKeyRef:
                  name: {{ $esUserSecret }}
                  key: ELASTICSEARCH_PASSWORD
+            - name: LDAP_URL
+              value: {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+            - name: BIND_DN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: BIND_DN
+            - name: BIND_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: BIND_PASSWORD
          volumeMounts:
            - name: elasticsearch-bin
              mountPath: /tmp/apache.sh
--- a/elasticsearch/templates/etc/_elasticsearch-host.conf.tpl
+++ b/elasticsearch/templates/etc/_elasticsearch-host.conf.tpl
@ -15,14 +15,20 @@ limitations under the License.
 */}}

 <VirtualHost *:80>
+  ProxyRequests On
+  ProxyPreserveHost On
  <Location />
      ProxyPass http://localhost:${ELASTICSEARCH_PORT}/
      ProxyPassReverse http://localhost:${ELASTICSEARCH_PORT}/
  </Location>
  <Proxy *>
+      AuthName "Elasticsearch"
      AuthType Basic
-      AuthName "Authentication Required for Elasticsearch"
-      AuthUserFile {{.Values.conf.apache.htpasswd | quote}}
+      AuthBasicProvider ldap file
+      AuthUserFile /usr/local/apache2/conf/.htpasswd
+      AuthLDAPBindDN ${BIND_DN}
+      AuthLDAPBindPassword ${BIND_PASSWORD}
+      AuthLDAPURL ${LDAP_URL}
      Require valid-user
  </Proxy>
 </VirtualHost>
--- a/elasticsearch/templates/etc/_httpd.conf.tpl
+++ b/elasticsearch/templates/etc/_httpd.conf.tpl
@ -52,6 +52,8 @@ LoadModule authz_user_module modules/mod_authz_user.so
 LoadModule authz_core_module modules/mod_authz_core.so
 LoadModule access_compat_module modules/mod_access_compat.so
 LoadModule auth_basic_module modules/mod_auth_basic.so
+LoadModule ldap_module modules/mod_ldap.so
+LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
 LoadModule reqtimeout_module modules/mod_reqtimeout.so
 LoadModule filter_module modules/mod_filter.so
 LoadModule proxy_html_module modules/mod_proxy_html.so
--- a/elasticsearch/templates/secret-elasticsearch.yaml
+++ b/elasticsearch/templates/secret-elasticsearch.yaml
@ -32,4 +32,6 @@ data:
  ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }}
  ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }}
  ELASTICSEARCH_URI: {{ $elasticsearch_uri | b64enc }}
+  BIND_DN: {{ .Values.endpoints.ldap.auth.admin.bind | b64enc }}
+  BIND_PASSWORD: {{ .Values.endpoints.ldap.auth.admin.password | b64enc }}
 {{- end }}
--- a/elasticsearch/values.yaml
+++ b/elasticsearch/values.yaml
@ -170,8 +170,8 @@ secrets:

 conf:
  apache:
-    htpasswd: /usr/local/apache2/conf/.htpasswd
-  httpd:
+    httpd: null
+    elasticsearch_host: null
  init:
    max_map_count: 262144
  curator:
@ -370,6 +370,22 @@ endpoints:
    port:
      metrics:
        default: 9108
+  ldap:
+    hosts:
+      default: ldap
+    auth:
+      admin:
+        bind: "cn=admin,dc=cluster,dc=local"
+        password: password
+    host_fqdn_override:
+      default: null
+    path:
+      default: "/ou=People,dc=cluster,dc=local"
+    scheme:
+      default: ldap
+    port:
+      ldap:
+        default: 389

 monitoring:
  prometheus:
--- a/kibana/templates/bin/_apache.sh.tpl
+++ b/kibana/templates/bin/_apache.sh.tpl
@ -29,12 +29,6 @@ function start () {
  # Apache gets grumpy about PID files pre-existing
  rm -f /etc/httpd/logs/httpd.pid

-  if [ -f {{ .Values.conf.apache.htpasswd }} ]; then
-    htpasswd -b {{ .Values.conf.apache.htpasswd }} $KIBANA_USERNAME $KIBANA_PASSWORD
-  else
-    htpasswd -cb {{ .Values.conf.apache.htpasswd }} $KIBANA_USERNAME $KIBANA_PASSWORD
-  fi
-
  #Launch Apache on Foreground
  exec httpd -DFOREGROUND
 }
--- a/kibana/templates/deployment.yaml
+++ b/kibana/templates/deployment.yaml
@ -67,6 +67,18 @@ spec:
                secretKeyRef:
                  name: {{ $esUserSecret }}
                  key: ELASTICSEARCH_PASSWORD
+            - name: LDAP_URL
+              value: {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+            - name: BIND_DN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: BIND_DN
+            - name: BIND_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: BIND_PASSWORD
          volumeMounts:
            - name: kibana-bin
              mountPath: /tmp/apache.sh
--- a/kibana/templates/etc/_httpd.conf.tpl
+++ b/kibana/templates/etc/_httpd.conf.tpl
@ -52,6 +52,8 @@ LoadModule authz_user_module modules/mod_authz_user.so
 LoadModule authz_core_module modules/mod_authz_core.so
 LoadModule access_compat_module modules/mod_access_compat.so
 LoadModule auth_basic_module modules/mod_auth_basic.so
+LoadModule ldap_module modules/mod_ldap.so
+LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
 LoadModule reqtimeout_module modules/mod_reqtimeout.so
 LoadModule filter_module modules/mod_filter.so
 LoadModule proxy_html_module modules/mod_proxy_html.so
--- a/kibana/templates/etc/_kibana-host.conf.tpl
+++ b/kibana/templates/etc/_kibana-host.conf.tpl
@ -20,9 +20,12 @@ limitations under the License.
      ProxyPassReverse http://localhost:${KIBANA_PORT}/
  </Location>
  <Proxy *>
+      AuthName "Kibana"
      AuthType Basic
-      AuthName "Authentication Required for Kibana"
-      AuthUserFile {{.Values.conf.apache.htpasswd | quote}}
+      AuthBasicProvider ldap
+      AuthLDAPBindDN ${BIND_DN}
+      AuthLDAPBindPassword ${BIND_PASSWORD}
+      AuthLDAPURL ${LDAP_URL}
      Require valid-user
  </Proxy>
 </VirtualHost>
--- a/kibana/templates/secret-elasticsearch-creds.yaml
+++ b/kibana/templates/secret-elasticsearch-creds.yaml
@ -26,4 +26,6 @@ type: Opaque
 data:
  ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }}
  ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }}
+  BIND_DN: {{ .Values.endpoints.ldap.auth.admin.bind | b64enc }}
+  BIND_PASSWORD: {{ .Values.endpoints.ldap.auth.admin.password | b64enc }}
 {{- end }}
--- a/kibana/values.yaml
+++ b/kibana/values.yaml
@ -93,8 +93,8 @@ dependencies:

 conf:
  apache:
-    htpasswd: /usr/local/apache2/conf/.htpasswd
-  httpd:
+    httpd: null
+    kibana_host: null
  kibana:
    elasticsearch:
      pingTimeout: 1500
@ -168,6 +168,22 @@ endpoints:
        default: 5601
      http:
        default: 80
+  ldap:
+    hosts:
+      default: ldap
+    auth:
+      admin:
+        bind: "cn=admin,dc=cluster,dc=local"
+        password: password
+    host_fqdn_override:
+      default: null
+    path:
+      default: "/ou=People,dc=cluster,dc=local"
+    scheme:
+      default: ldap
+    port:
+      ldap:
+        default: 389

 network:
  kibana:
--- a/playbooks/osh-infra-dev-deploy.yaml
+++ b/playbooks/osh-infra-dev-deploy.yaml
@ -96,6 +96,12 @@
        ./tools/deployment/developer/110-elasticsearch.sh
      args:
        chdir: "{{ zuul.project.src_dir }}"
+    - name: Test LDAP Auth for Elasticsearch
+      shell: |
+        set -xe;
+        ./tools/deployment/developer/115-elasticsearch-ldap.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
    - name: Deploy Fluent-Logging
      shell: |
        set -xe;
--- a/playbooks/osh-infra-ldap-deploy.yaml
+++ b/playbooks/osh-infra-ldap-deploy.yaml
@ -0,0 +1,58 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Deploy Required packages
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/000-install-packages.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy Kubernetes
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/010-deploy-k8s.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy NFS for Logging, Monitoring and Alerting Components
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/020-lma-nfs-provisioner.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy LDAP
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/030-ldap.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy Elasticsearch
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/040-elasticsearch.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Test Elasticsearch Access via LDAP
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/045-elasticsearch-ldap.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy Kibana
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/050-kibana.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
--- a/playbooks/osh-infra-multinode-deploy.yaml
+++ b/playbooks/osh-infra-multinode-deploy.yaml
@ -80,6 +80,12 @@
        ./tools/deployment/multinode/110-elasticsearch.sh
      args:
        chdir: "{{ zuul.project.src_dir }}"
+    - name: Test LDAP Auth for Elasticsearch
+      shell: |
+        set -xe;
+        ./tools/deployment/multinode/115-elasticsearch-ldap.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
    - name: Deploy Fluent-Logging
      shell: |
        set -xe;
--- a/tools/deployment/common/115-elasticsearch-ldap.sh
+++ b/tools/deployment/common/115-elasticsearch-ldap.sh
@ -0,0 +1,91 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+ELASTICSEARCH_ENDPOINT="elasticsearch-logging.openstack"
+
+#NOTE: Create index with specified LDAP user
+function create_index () {
+  index_result=$(curl -K- <<< "--user $1:$2" \
+  -XPUT "${ELASTICSEARCH_ENDPOINT}/$1_index?pretty" -H 'Content-Type: application/json' -d'
+  {
+    "settings" : {
+      "index" : {
+        "number_of_shards" : 3,
+        "number_of_replicas" : 2
+      }
+    }
+  }
+  ' | python -c "import sys, json; print json.load(sys.stdin)['acknowledged']")
+  if [ "$index_result" == "True" ];
+  then
+     echo "$1's index successfully created!";
+  else
+     echo "$1's index not created!";
+     exit 1;
+  fi
+}
+
+#NOTE: Insert test data with specified LDAP user
+function insert_test_data () {
+  insert_result=$(curl -K- <<< "--user $1:$2" \
+  -XPUT "${ELASTICSEARCH_ENDPOINT}/$1_index/sample_type/123/_create?pretty" -H 'Content-Type: application/json' -d'
+  {
+      "name" : "Elasticsearch",
+      "message" : "Test data text entry"
+  }
+  ' | python -c "import sys, json; print json.load(sys.stdin)['result']")
+  if [ "$insert_result" == "created" ]; then
+     sleep 20
+     echo "Test data inserted into $1's index!";
+  else
+     echo "Test data not inserted into $1's index!";
+     exit 1;
+  fi
+}
+
+#NOTE: Check hits on test data in specified LDAP user's index
+function check_hits () {
+  total_hits=$(curl -K- <<< "--user $1:$2" \
+  "${ELASTICSEARCH_ENDPOINT}/_search?pretty" -H 'Content-Type: application/json' -d'
+  {
+    "query" : {
+      "bool": {
+        "must": [
+          { "match": { "name": "Elasticsearch" }},
+          { "match": { "message": "Test data text entry" }}
+        ]
+      }
+    }
+  }
+  ' | python -c "import sys, json; print json.load(sys.stdin)['hits']['total']")
+  if [ "$total_hits" -gt 0 ]; then
+     echo "Successful hits on test data query on $1's index!"
+  else
+     echo "No hits on query for test data on $1's index!";
+     exit 1;
+  fi
+}
+
+create_index bob password
+create_index alice password
+
+insert_test_data bob password
+insert_test_data alice password
+
+check_hits bob password
+check_hits alice password
--- a/tools/deployment/developer/000-install-packages.sh
+++ b/tools/deployment/developer/000-install-packages.sh
@ -1 +0,0 @@
-../common/000-install-packages.sh
--- a/tools/deployment/developer/000-install-packages.sh
+++ b/tools/deployment/developer/000-install-packages.sh
@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+sudo apt-get update
+sudo apt-get install --no-install-recommends -y \
+        ca-certificates \
+        git \
+        make \
+        nmap \
+        curl
--- a/tools/deployment/developer/005-deploy-k8s.sh
+++ b/tools/deployment/developer/005-deploy-k8s.sh
@ -1 +0,0 @@
-../common/005-deploy-k8s.sh
--- a/tools/deployment/developer/005-deploy-k8s.sh
+++ b/tools/deployment/developer/005-deploy-k8s.sh
@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+make dev-deploy setup-host
+make dev-deploy k8s
--- a/tools/deployment/developer/115-elasticsearch-ldap.sh
+++ b/tools/deployment/developer/115-elasticsearch-ldap.sh
@ -0,0 +1 @@
+../common/115-elasticsearch-ldap.sh
--- a/tools/deployment/multinode/115-elasticsearch-ldap.sh
+++ b/tools/deployment/multinode/115-elasticsearch-ldap.sh
@ -0,0 +1 @@
+../common/115-elasticsearch-ldap.sh