From 3c692abd6ede7b831cce2f1e5cf046cf842c1ef5 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Fri, 20 Apr 2018 14:54:05 -0500 Subject: [PATCH] Add ldap support in elasticsearch/kibana apache proxies This adds required configuration for enabling LDAP through the apache proxy in the elasticsearch and kibana charts by default Change-Id: Iaff8f328ff50944ddad94ec86b1134ca73750176 --- elasticsearch/templates/bin/_apache.sh.tpl | 6 +- .../templates/deployment-client.yaml | 12 +++ .../etc/_elasticsearch-host.conf.tpl | 10 +- elasticsearch/templates/etc/_httpd.conf.tpl | 2 + .../templates/secret-elasticsearch.yaml | 2 + elasticsearch/values.yaml | 20 +++- kibana/templates/bin/_apache.sh.tpl | 6 -- kibana/templates/deployment.yaml | 12 +++ kibana/templates/etc/_httpd.conf.tpl | 2 + kibana/templates/etc/_kibana-host.conf.tpl | 7 +- .../templates/secret-elasticsearch-creds.yaml | 2 + kibana/values.yaml | 20 +++- playbooks/osh-infra-dev-deploy.yaml | 6 ++ playbooks/osh-infra-ldap-deploy.yaml | 58 ++++++++++++ playbooks/osh-infra-multinode-deploy.yaml | 6 ++ .../common/115-elasticsearch-ldap.sh | 91 +++++++++++++++++++ .../developer/000-install-packages.sh | 26 +++++- tools/deployment/developer/005-deploy-k8s.sh | 21 ++++- .../developer/115-elasticsearch-ldap.sh | 1 + .../multinode/115-elasticsearch-ldap.sh | 1 + 20 files changed, 292 insertions(+), 19 deletions(-) create mode 100644 playbooks/osh-infra-ldap-deploy.yaml create mode 100755 tools/deployment/common/115-elasticsearch-ldap.sh mode change 120000 => 100755 tools/deployment/developer/000-install-packages.sh mode change 120000 => 100755 tools/deployment/developer/005-deploy-k8s.sh create mode 120000 tools/deployment/developer/115-elasticsearch-ldap.sh create mode 120000 tools/deployment/multinode/115-elasticsearch-ldap.sh diff --git a/elasticsearch/templates/bin/_apache.sh.tpl b/elasticsearch/templates/bin/_apache.sh.tpl index b03ac0945..cbba386da 100644 --- a/elasticsearch/templates/bin/_apache.sh.tpl +++ b/elasticsearch/templates/bin/_apache.sh.tpl @@ -29,10 +29,10 @@ function start () { # Apache gets grumpy about PID files pre-existing rm -f /etc/httpd/logs/httpd.pid - if [ -f {{ .Values.conf.apache.htpasswd }} ]; then - htpasswd -b {{ .Values.conf.apache.htpasswd }} $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD + if [ -f /usr/local/apache2/conf/.htpasswd ]; then + htpasswd -b /usr/local/apache2/conf/.htpasswd $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD else - htpasswd -cb {{ .Values.conf.apache.htpasswd }} $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD + htpasswd -cb /usr/local/apache2/conf/.htpasswd $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD fi #Launch Apache on Foreground diff --git a/elasticsearch/templates/deployment-client.yaml b/elasticsearch/templates/deployment-client.yaml index c50549682..4cf607505 100644 --- a/elasticsearch/templates/deployment-client.yaml +++ b/elasticsearch/templates/deployment-client.yaml @@ -129,6 +129,18 @@ spec: secretKeyRef: name: {{ $esUserSecret }} key: ELASTICSEARCH_PASSWORD + - name: LDAP_URL + value: {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} + - name: BIND_DN + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: BIND_DN + - name: BIND_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: BIND_PASSWORD volumeMounts: - name: elasticsearch-bin mountPath: /tmp/apache.sh diff --git a/elasticsearch/templates/etc/_elasticsearch-host.conf.tpl b/elasticsearch/templates/etc/_elasticsearch-host.conf.tpl index 8b7a3207f..6ead2d76c 100644 --- a/elasticsearch/templates/etc/_elasticsearch-host.conf.tpl +++ b/elasticsearch/templates/etc/_elasticsearch-host.conf.tpl @@ -15,14 +15,20 @@ limitations under the License. */}} + ProxyRequests On + ProxyPreserveHost On ProxyPass http://localhost:${ELASTICSEARCH_PORT}/ ProxyPassReverse http://localhost:${ELASTICSEARCH_PORT}/ + AuthName "Elasticsearch" AuthType Basic - AuthName "Authentication Required for Elasticsearch" - AuthUserFile {{.Values.conf.apache.htpasswd | quote}} + AuthBasicProvider ldap file + AuthUserFile /usr/local/apache2/conf/.htpasswd + AuthLDAPBindDN ${BIND_DN} + AuthLDAPBindPassword ${BIND_PASSWORD} + AuthLDAPURL ${LDAP_URL} Require valid-user diff --git a/elasticsearch/templates/etc/_httpd.conf.tpl b/elasticsearch/templates/etc/_httpd.conf.tpl index 1cd54e976..19af85523 100644 --- a/elasticsearch/templates/etc/_httpd.conf.tpl +++ b/elasticsearch/templates/etc/_httpd.conf.tpl @@ -52,6 +52,8 @@ LoadModule authz_user_module modules/mod_authz_user.so LoadModule authz_core_module modules/mod_authz_core.so LoadModule access_compat_module modules/mod_access_compat.so LoadModule auth_basic_module modules/mod_auth_basic.so +LoadModule ldap_module modules/mod_ldap.so +LoadModule authnz_ldap_module modules/mod_authnz_ldap.so LoadModule reqtimeout_module modules/mod_reqtimeout.so LoadModule filter_module modules/mod_filter.so LoadModule proxy_html_module modules/mod_proxy_html.so diff --git a/elasticsearch/templates/secret-elasticsearch.yaml b/elasticsearch/templates/secret-elasticsearch.yaml index 91d3f15e8..0f5b17611 100644 --- a/elasticsearch/templates/secret-elasticsearch.yaml +++ b/elasticsearch/templates/secret-elasticsearch.yaml @@ -32,4 +32,6 @@ data: ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} ELASTICSEARCH_URI: {{ $elasticsearch_uri | b64enc }} + BIND_DN: {{ .Values.endpoints.ldap.auth.admin.bind | b64enc }} + BIND_PASSWORD: {{ .Values.endpoints.ldap.auth.admin.password | b64enc }} {{- end }} diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 95f2e26b9..fd274c05a 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -170,8 +170,8 @@ secrets: conf: apache: - htpasswd: /usr/local/apache2/conf/.htpasswd - httpd: + httpd: null + elasticsearch_host: null init: max_map_count: 262144 curator: @@ -370,6 +370,22 @@ endpoints: port: metrics: default: 9108 + ldap: + hosts: + default: ldap + auth: + admin: + bind: "cn=admin,dc=cluster,dc=local" + password: password + host_fqdn_override: + default: null + path: + default: "/ou=People,dc=cluster,dc=local" + scheme: + default: ldap + port: + ldap: + default: 389 monitoring: prometheus: diff --git a/kibana/templates/bin/_apache.sh.tpl b/kibana/templates/bin/_apache.sh.tpl index d8892177d..e80ead098 100644 --- a/kibana/templates/bin/_apache.sh.tpl +++ b/kibana/templates/bin/_apache.sh.tpl @@ -29,12 +29,6 @@ function start () { # Apache gets grumpy about PID files pre-existing rm -f /etc/httpd/logs/httpd.pid - if [ -f {{ .Values.conf.apache.htpasswd }} ]; then - htpasswd -b {{ .Values.conf.apache.htpasswd }} $KIBANA_USERNAME $KIBANA_PASSWORD - else - htpasswd -cb {{ .Values.conf.apache.htpasswd }} $KIBANA_USERNAME $KIBANA_PASSWORD - fi - #Launch Apache on Foreground exec httpd -DFOREGROUND } diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml index ac8e788ac..d46d3abb7 100644 --- a/kibana/templates/deployment.yaml +++ b/kibana/templates/deployment.yaml @@ -67,6 +67,18 @@ spec: secretKeyRef: name: {{ $esUserSecret }} key: ELASTICSEARCH_PASSWORD + - name: LDAP_URL + value: {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} + - name: BIND_DN + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: BIND_DN + - name: BIND_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: BIND_PASSWORD volumeMounts: - name: kibana-bin mountPath: /tmp/apache.sh diff --git a/kibana/templates/etc/_httpd.conf.tpl b/kibana/templates/etc/_httpd.conf.tpl index 1cd54e976..19af85523 100644 --- a/kibana/templates/etc/_httpd.conf.tpl +++ b/kibana/templates/etc/_httpd.conf.tpl @@ -52,6 +52,8 @@ LoadModule authz_user_module modules/mod_authz_user.so LoadModule authz_core_module modules/mod_authz_core.so LoadModule access_compat_module modules/mod_access_compat.so LoadModule auth_basic_module modules/mod_auth_basic.so +LoadModule ldap_module modules/mod_ldap.so +LoadModule authnz_ldap_module modules/mod_authnz_ldap.so LoadModule reqtimeout_module modules/mod_reqtimeout.so LoadModule filter_module modules/mod_filter.so LoadModule proxy_html_module modules/mod_proxy_html.so diff --git a/kibana/templates/etc/_kibana-host.conf.tpl b/kibana/templates/etc/_kibana-host.conf.tpl index 6041c803c..a58e00bd7 100644 --- a/kibana/templates/etc/_kibana-host.conf.tpl +++ b/kibana/templates/etc/_kibana-host.conf.tpl @@ -20,9 +20,12 @@ limitations under the License. ProxyPassReverse http://localhost:${KIBANA_PORT}/ + AuthName "Kibana" AuthType Basic - AuthName "Authentication Required for Kibana" - AuthUserFile {{.Values.conf.apache.htpasswd | quote}} + AuthBasicProvider ldap + AuthLDAPBindDN ${BIND_DN} + AuthLDAPBindPassword ${BIND_PASSWORD} + AuthLDAPURL ${LDAP_URL} Require valid-user diff --git a/kibana/templates/secret-elasticsearch-creds.yaml b/kibana/templates/secret-elasticsearch-creds.yaml index 0ea91703f..11db0eb94 100644 --- a/kibana/templates/secret-elasticsearch-creds.yaml +++ b/kibana/templates/secret-elasticsearch-creds.yaml @@ -26,4 +26,6 @@ type: Opaque data: ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} + BIND_DN: {{ .Values.endpoints.ldap.auth.admin.bind | b64enc }} + BIND_PASSWORD: {{ .Values.endpoints.ldap.auth.admin.password | b64enc }} {{- end }} diff --git a/kibana/values.yaml b/kibana/values.yaml index b42ce08a3..3a02215d5 100644 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -93,8 +93,8 @@ dependencies: conf: apache: - htpasswd: /usr/local/apache2/conf/.htpasswd - httpd: + httpd: null + kibana_host: null kibana: elasticsearch: pingTimeout: 1500 @@ -168,6 +168,22 @@ endpoints: default: 5601 http: default: 80 + ldap: + hosts: + default: ldap + auth: + admin: + bind: "cn=admin,dc=cluster,dc=local" + password: password + host_fqdn_override: + default: null + path: + default: "/ou=People,dc=cluster,dc=local" + scheme: + default: ldap + port: + ldap: + default: 389 network: kibana: diff --git a/playbooks/osh-infra-dev-deploy.yaml b/playbooks/osh-infra-dev-deploy.yaml index 7da09e263..45a16c21a 100644 --- a/playbooks/osh-infra-dev-deploy.yaml +++ b/playbooks/osh-infra-dev-deploy.yaml @@ -96,6 +96,12 @@ ./tools/deployment/developer/110-elasticsearch.sh args: chdir: "{{ zuul.project.src_dir }}" + - name: Test LDAP Auth for Elasticsearch + shell: | + set -xe; + ./tools/deployment/developer/115-elasticsearch-ldap.sh + args: + chdir: "{{ zuul.project.src_dir }}" - name: Deploy Fluent-Logging shell: | set -xe; diff --git a/playbooks/osh-infra-ldap-deploy.yaml b/playbooks/osh-infra-ldap-deploy.yaml new file mode 100644 index 000000000..7df5788ae --- /dev/null +++ b/playbooks/osh-infra-ldap-deploy.yaml @@ -0,0 +1,58 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: primary + tasks: + - name: Deploy Required packages + shell: | + set -xe; + ./tools/deployment/ldap/000-install-packages.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Kubernetes + shell: | + set -xe; + ./tools/deployment/ldap/010-deploy-k8s.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy NFS for Logging, Monitoring and Alerting Components + shell: | + set -xe; + ./tools/deployment/ldap/020-lma-nfs-provisioner.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy LDAP + shell: | + set -xe; + ./tools/deployment/ldap/030-ldap.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Elasticsearch + shell: | + set -xe; + ./tools/deployment/ldap/040-elasticsearch.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Test Elasticsearch Access via LDAP + shell: | + set -xe; + ./tools/deployment/ldap/045-elasticsearch-ldap.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Kibana + shell: | + set -xe; + ./tools/deployment/ldap/050-kibana.sh + args: + chdir: "{{ zuul.project.src_dir }}" diff --git a/playbooks/osh-infra-multinode-deploy.yaml b/playbooks/osh-infra-multinode-deploy.yaml index 958659779..d82bccc70 100644 --- a/playbooks/osh-infra-multinode-deploy.yaml +++ b/playbooks/osh-infra-multinode-deploy.yaml @@ -80,6 +80,12 @@ ./tools/deployment/multinode/110-elasticsearch.sh args: chdir: "{{ zuul.project.src_dir }}" + - name: Test LDAP Auth for Elasticsearch + shell: | + set -xe; + ./tools/deployment/multinode/115-elasticsearch-ldap.sh + args: + chdir: "{{ zuul.project.src_dir }}" - name: Deploy Fluent-Logging shell: | set -xe; diff --git a/tools/deployment/common/115-elasticsearch-ldap.sh b/tools/deployment/common/115-elasticsearch-ldap.sh new file mode 100755 index 000000000..830a012a4 --- /dev/null +++ b/tools/deployment/common/115-elasticsearch-ldap.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +ELASTICSEARCH_ENDPOINT="elasticsearch-logging.openstack" + +#NOTE: Create index with specified LDAP user +function create_index () { + index_result=$(curl -K- <<< "--user $1:$2" \ + -XPUT "${ELASTICSEARCH_ENDPOINT}/$1_index?pretty" -H 'Content-Type: application/json' -d' + { + "settings" : { + "index" : { + "number_of_shards" : 3, + "number_of_replicas" : 2 + } + } + } + ' | python -c "import sys, json; print json.load(sys.stdin)['acknowledged']") + if [ "$index_result" == "True" ]; + then + echo "$1's index successfully created!"; + else + echo "$1's index not created!"; + exit 1; + fi +} + +#NOTE: Insert test data with specified LDAP user +function insert_test_data () { + insert_result=$(curl -K- <<< "--user $1:$2" \ + -XPUT "${ELASTICSEARCH_ENDPOINT}/$1_index/sample_type/123/_create?pretty" -H 'Content-Type: application/json' -d' + { + "name" : "Elasticsearch", + "message" : "Test data text entry" + } + ' | python -c "import sys, json; print json.load(sys.stdin)['result']") + if [ "$insert_result" == "created" ]; then + sleep 20 + echo "Test data inserted into $1's index!"; + else + echo "Test data not inserted into $1's index!"; + exit 1; + fi +} + +#NOTE: Check hits on test data in specified LDAP user's index +function check_hits () { + total_hits=$(curl -K- <<< "--user $1:$2" \ + "${ELASTICSEARCH_ENDPOINT}/_search?pretty" -H 'Content-Type: application/json' -d' + { + "query" : { + "bool": { + "must": [ + { "match": { "name": "Elasticsearch" }}, + { "match": { "message": "Test data text entry" }} + ] + } + } + } + ' | python -c "import sys, json; print json.load(sys.stdin)['hits']['total']") + if [ "$total_hits" -gt 0 ]; then + echo "Successful hits on test data query on $1's index!" + else + echo "No hits on query for test data on $1's index!"; + exit 1; + fi +} + +create_index bob password +create_index alice password + +insert_test_data bob password +insert_test_data alice password + +check_hits bob password +check_hits alice password diff --git a/tools/deployment/developer/000-install-packages.sh b/tools/deployment/developer/000-install-packages.sh deleted file mode 120000 index d702c4899..000000000 --- a/tools/deployment/developer/000-install-packages.sh +++ /dev/null @@ -1 +0,0 @@ -../common/000-install-packages.sh \ No newline at end of file diff --git a/tools/deployment/developer/000-install-packages.sh b/tools/deployment/developer/000-install-packages.sh new file mode 100755 index 000000000..4b3129b07 --- /dev/null +++ b/tools/deployment/developer/000-install-packages.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +sudo apt-get update +sudo apt-get install --no-install-recommends -y \ + ca-certificates \ + git \ + make \ + nmap \ + curl diff --git a/tools/deployment/developer/005-deploy-k8s.sh b/tools/deployment/developer/005-deploy-k8s.sh deleted file mode 120000 index 257a39f7a..000000000 --- a/tools/deployment/developer/005-deploy-k8s.sh +++ /dev/null @@ -1 +0,0 @@ -../common/005-deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/developer/005-deploy-k8s.sh b/tools/deployment/developer/005-deploy-k8s.sh new file mode 100755 index 000000000..b0a3e8cc8 --- /dev/null +++ b/tools/deployment/developer/005-deploy-k8s.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +make dev-deploy setup-host +make dev-deploy k8s diff --git a/tools/deployment/developer/115-elasticsearch-ldap.sh b/tools/deployment/developer/115-elasticsearch-ldap.sh new file mode 120000 index 000000000..554cc7f26 --- /dev/null +++ b/tools/deployment/developer/115-elasticsearch-ldap.sh @@ -0,0 +1 @@ +../common/115-elasticsearch-ldap.sh \ No newline at end of file diff --git a/tools/deployment/multinode/115-elasticsearch-ldap.sh b/tools/deployment/multinode/115-elasticsearch-ldap.sh new file mode 120000 index 000000000..554cc7f26 --- /dev/null +++ b/tools/deployment/multinode/115-elasticsearch-ldap.sh @@ -0,0 +1 @@ +../common/115-elasticsearch-ldap.sh \ No newline at end of file