Add LDAP-backed domain gate

This patch set adds a nv-gating with an OpenLDAP server with some sample
data loaded for development or testing use using a bootstrap job.
This patch set also adds confirming authentication works using  domain-
specific configuration for keystone.

Consolidated change from: https://review.openstack.org/#/c/552976/
Co-Authored-By: Gage Hugo <gagehugo@gmail.com>

Change-Id: I1aeccffc018d0fcefc8e2b15a4ac6b83cb2be8b6
Signed-off-by: Tin Lam <tin@irrational.io>

.zuul.yaml

@@ -38,6 +38,12 @@
               - ^.*\.rst$
               - ^doc/.*$
               - ^releasenotes/.*$
+        - openstack-helm-dev-deploy-nfs-ldap:
+            voting: false
+            irrelevant-files:
+              - ^.*\.rst$
+              - ^doc/.*$
+              - ^releasenotes/.*$
         - openstack-helm-multinode-ubuntu:
             irrelevant-files:
               - ^.*\.rst$
@@ -158,6 +164,13 @@
     parent: openstack-helm-dev-deploy
     run: tools/gate/playbooks/dev-deploy-nfs.yaml
 
+- job:
+    name: openstack-helm-dev-deploy-nfs-ldap
+    vars:
+      idp_backend: ldap
+    parent: openstack-helm-dev-deploy
+    run: tools/gate/playbooks/dev-deploy-nfs.yaml
+
 - job:
     name: openstack-helm-multinode
     timeout: 7200

helm-toolkit/templates/manifests/_job-bootstrap.yaml

@@ -30,6 +30,7 @@ limitations under the License.
 {{- $configMapEtc := index . "configMapEtc" | default (printf "%s-%s" $serviceName "etc" ) -}}
 {{- $configFile := index . "configFile" | default (printf "/etc/%s/%s.conf" $serviceName $serviceName ) -}}
 {{- $keystoneUser := index . "keystoneUser" | default $serviceName -}}
+{{- $openrc := index . "openrc" | default "true" -}}
 
 {{- $serviceNamePretty := $serviceName | replace "_" "-" -}}
 
@@ -57,9 +58,11 @@ spec:
           image: {{ $envAll.Values.images.tags.bootstrap }}
           imagePullPolicy: {{ $envAll.Values.images.pull_policy }}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{- if eq $openrc "true" }}
           env:
 {{- with $env := dict "ksUserSecret" ( index $envAll.Values.secrets.identity $keystoneUser ) }}
 {{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }}
+{{- end }}
 {{- end }}
           command:
             - /tmp/bootstrap.sh

ldap/templates/_helpers.tpl

@@ -14,3 +14,9 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- $name := default .Chart.Name .Values.nameOverride -}}
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
+
+{{- define "splitdomain" -}}
+{{- $name := index . 0 -}}
+{{- $local := dict "first" true }}
+{{- range $k, $v := splitList "." $name }}{{- if not $local.first -}},{{- end -}}dc={{- $v -}}{{- $_ := set $local "first" false -}}{{- end -}}
+{{- end -}}

ldap/templates/bin/_bootstrap.sh.tpl

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -xe
+
+{{- $url := tuple "ldap" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $port := tuple "ldap" "internal" "ldap" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+LDAPHOST="ldap://{{ $url }}:{{ $port }}"
+ADMIN="cn={{ .Values.secrets.identity.admin }},{{ tuple .Values.openldap.domain . | include "splitdomain" }}"
+ldapadd -x -D $ADMIN -H $LDAPHOST -w {{ .Values.openldap.password }} -f /etc/sample_data.ldif

ldap/templates/configmap-bin.yaml

@@ -0,0 +1,27 @@
+{{/*
+Copyright 2018 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+{{- if .Values.manifests.configmap_bin }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ldap-bin
+data:
+{{- if .Values.bootstrap.enabled }}
+  bootstrap.sh: |
+{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
+{{- end }}

ldap/templates/configmap-etc.yaml

@@ -0,0 +1,27 @@
+{{/*
+Copyright 2018 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+{{- if .Values.manifests.configmap_etc }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ldap-etc
+data:
+{{- if .Values.bootstrap.enabled }}
+  sample_data.ldif: |
+{{ .Values.data.sample | indent 4 }}
+{{- end }}
+{{- end }}

ldap/templates/job-bootstrap.yaml

@@ -0,0 +1,18 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.job_bootstrap .Values.bootstrap.enabled }}
+{{- $bootstrapJob := dict "envAll" . "serviceName" "ldap" "configFile" "/etc/sample_data.ldif" "keystoneUser" "admin" "openrc" "false" -}}
+{{ $bootstrapJob | include "helm-toolkit.manifests.job_bootstrap" }}
+{{- end }}

ldap/values.yaml

@@ -25,6 +25,14 @@ pod:
         default: kubernetes.io/hostname
   replicas:
     server: 1
+  lifecycle:
+    upgrades:
+      deployments:
+        revision_history: 3
+        pod_replacement_strategy: RollingUpdate
+        rolling_update:
+          max_unavailable: 1
+          max_surge: 3
   resources:
     enabled: false
     server:
@@ -34,9 +42,23 @@ pod:
       limits:
         memory: "1024Mi"
         cpu: "2000m"
+    jobs:
+      bootstrap:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+  mounts:
+    ldap_data_load:
+      init_container: null
+      ldap_data_load:
+
 images:
   tags:
-    ldap: "docker.io/osixia/openldap:1.1.9"
+    bootstrap: "docker.io/osixia/openldap:1.2.0"
+    ldap: "docker.io/osixia/openldap:1.2.0"
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
   pull_policy: IfNotPresent
 
@@ -44,6 +66,10 @@ dependencies:
   static:
     ldap:
       jobs: null
+    bootstrap:
+      services:
+        - endpoint: internal
+          service: ldap
 
 storage:
   pvc:
@@ -58,6 +84,12 @@ labels:
   server:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
+  job:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+
+bootstrap:
+  enabled: false
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -72,10 +104,98 @@ endpoints:
       ldap:
         default: 389
 
+data:
+  sample: |
+    dn: ou=People,dc=cluster,dc=local
+    objectclass: organizationalunit
+    ou: People
+    description: We the People
+
+    # NOTE: Password is "password" without quotes
+    dn: uid=alice,ou=People,dc=cluster,dc=local
+    objectClass: inetOrgPerson
+    objectClass: top
+    objectClass: posixAccount
+    objectClass: shadowAccount
+    objectClass: person
+    sn: Alice
+    cn: alice
+    uid: alice
+    userPassword: {SSHA}+i3t/DLCgLDGaIOAmfeFJ2kDeJWmPUDH
+    description: SHA
+    gidNumber: 1000
+    uidNumber: 1493
+    homeDirectory: /home/alice
+    mail: alice@example.com
+
+    # NOTE: Password is "password" without quotes
+    dn: uid=bob,ou=People,dc=cluster,dc=local
+    objectClass: inetOrgPerson
+    objectClass: top
+    objectClass: posixAccount
+    objectClass: shadowAccount
+    objectClass: person
+    sn: Bob
+    cn: bob
+    uid: bob
+    userPassword: {SSHA}fCJ5vuW1BQ4/OfOVkkx1qjwi7yHFuGNB
+    description: MD5
+    gidNumber: 1000
+    uidNumber: 5689
+    homeDirectory: /home/bob
+    mail: bob@example.com
+
+    dn: ou=Groups,dc=cluster,dc=local
+    objectclass: organizationalunit
+    ou: Groups
+    description: We the People
+
+    dn: cn=cryptography,ou=Groups,dc=cluster,dc=local
+    objectclass: top
+    objectclass: posixGroup
+    gidNumber: 418
+    cn: overwatch
+    description: Cryptography Team
+    memberUID: uid=alice,ou=People,dc=cluster,dc=local
+    memberUID: uid=bob,ou=People,dc=cluster,dc=local
+
+    dn: cn=blue,ou=Groups,dc=cluster,dc=local
+    objectclass: top
+    objectclass: posixGroup
+    gidNumber: 419
+    cn: blue
+    description: Blue Team
+    memberUID: uid=bob,ou=People,dc=cluster,dc=local
+
+    dn: cn=red,ou=Groups,dc=cluster,dc=local
+    objectclass: top
+    objectclass: posixGroup
+    gidNumber: 420
+    cn: red
+    description: Red Team
+    memberUID: uid=alice,ou=People,dc=cluster,dc=local
+
+secrets:
+  identity:
+    admin: admin
+    ldap: ldap
+
+dependencies:
+  static:
+    server:
+      jobs:
+        - ldap-load-data
+      services:
+        - endpoint: internal
+          service: ldap
+
 openldap:
   domain: cluster.local
   password: password
 
 manifests:
+  configmap_bin: true
+  configmap_etc: true
+  job_bootstrap: true
   statefulset: true
   service: true

tools/deployment/developer/ldap/080-keystone.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Handle LDAP
+make pull-images ldap
+
+#NOTE: Deploy command
+: ${OSH_EXTRA_HELM_ARGS:=""}
+helm upgrade --install ldap ./ldap \
+    --namespace=openstack \
+    --set pod.replicas.server=1 \
+    --set bootstrap.enabled=true \
+    ${OSH_EXTRA_HELM_ARGS} \
+    ${OSH_EXTRA_HELM_ARGS_LDAP}
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh openstack
+
+#NOTE: Validate Deployment info
+helm status ldap
+
+#NOTE: Handle Keystone
+make pull-images keystone
+
+#NOTE: Deploy command
+: ${OSH_EXTRA_HELM_ARGS:=""}
+helm upgrade --install keystone ./keystone \
+    --namespace=openstack \
+    --values=./tools/overrides/keystone/ldap_domain_config.yaml \
+    ${OSH_EXTRA_HELM_ARGS} \
+    ${OSH_EXTRA_HELM_ARGS_KEYSTONE}
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh openstack
+
+#NOTE: Validate Deployment info
+helm status keystone
+export OS_CLOUD=openstack_helm
+sleep 30 #NOTE(portdirect): Wait for ingress controller to update rules and restart Nginx
+openstack endpoint list
+
+#NOTE: Do some additional queries here for LDAP
+openstack domain list
+openstack user list
+openstack user list --domain ldapdomain
+
+openstack role add --user bob --project admin --user-domain ldapdomain --project-domain default admin
+
+#NOTE: Testing we can auth against the LDAP user
+unset OS_CLOUD
+openstack --os-auth-url http://keystone.openstack.svc.cluster.local/v3 --os-username bob --os-password password --os-user-domain-name ldapdomain --os-identity-api-version 3 token issue

tools/gate/playbooks/dev-deploy-nfs.yaml

@@ -87,6 +87,7 @@
       args:
         chdir: "{{ zuul.project.src_dir }}"
     - name: Deploy Keystone
+      when: idp_backend is not defined
       shell: |
         set -xe;
         ./tools/deployment/developer/nfs/080-keystone.sh
@@ -95,6 +96,16 @@
         OSH_INFRA_PATH: "{{ zuul_osh_infra_relative_path | default('') }}"
       args:
         chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy Keystone with LDAP
+      when: idp_backend is defined and idp_backend == "ldap"
+      shell: |
+        set -xe;
+        ./tools/deployment/developer/ldap/080-keystone.sh
+      environment:
+        OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+        OSH_INFRA_PATH: "{{ zuul_osh_infra_relative_path | default('') }}"
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
     - name: Deploy Heat
       shell: |
         set -xe;

tools/overrides/example/keystone_domain_config.yaml

@@ -1,49 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This example sets the default domain to be LDAP based, and adds in a new
-# dbdomain that is SQL-backed.  Note that for this to work, you need to set
-# an admin (env: OS_USERNAME and OS_PASSWORD) that is valid in the LDAP.
-conf:
-  keystone:
-    identity:
-      driver: ldap
-      default_domain_id: default
-      domain_specific_drivers_enabled: True
-      domain_configurations_from_database: True
-      domain_config_dir: /etc/keystonedomains
-    ldap:
-      url: "ldap://ldap.openstack.svc.cluster.local:389"
-      user: "cn=admin,dc=cluster,dc=local"
-      password: password
-      suffix: "dc=cluster,dc=local"
-      user_attribute_ignore: enabled,email,tenants,default_project_id
-      query_scope: sub
-      user_enabled_emulation: True
-      user_enabled_emulation_dn: "cn=overwatch,ou=Groups,dc=cluster,dc=local"
-      user_tree_dn: "ou=People,dc=cluster,dc=local"
-      user_enabled_mask: 2
-      user_enabled_default: 512
-      user_name_attribute: cn
-      user_id_attribute: sn
-      user_mail_attribute: mail
-      user_pass_attribute: userPassword
-      group_tree_dn: "ou=Groups,dc=cluster,dc=local"
-      user_allow_create: False
-      user_allow_delete: False
-      user_allow_update: False
-  ks_domains:
-    dbdomain:
-      identity:
-        driver: sql

tools/overrides/keystone/ldap_domain_config.yaml

@@ -0,0 +1,46 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+conf:
+  keystone:
+    identity:
+      driver: sql
+      default_domain_id: default
+      domain_specific_drivers_enabled: True
+      domain_configurations_from_database: True
+      domain_config_dir: /etc/keystonedomains
+  ks_domains:
+    ldapdomain:
+      identity:
+        driver: ldap
+      ldap:
+        url: "ldap://ldap.openstack.svc.cluster.local:389"
+        user: "cn=admin,dc=cluster,dc=local"
+        password: password
+        suffix: "dc=cluster,dc=local"
+        user_attribute_ignore: "enabled,email,tenants,default_project_id"
+        query_scope: sub
+        user_enabled_emulation: True
+        user_enabled_emulation_dn: "cn=overwatch,ou=Groups,dc=cluster,dc=local"
+        user_tree_dn: "ou=People,dc=cluster,dc=local"
+        user_enabled_mask: 2
+        user_enabled_default: 512
+        user_name_attribute: cn
+        user_id_attribute: sn
+        user_mail_attribute: mail
+        user_pass_attribute: userPassword
+        group_tree_dn: "ou=Groups,dc=cluster,dc=local"
+        user_allow_create: False
+        user_allow_delete: False
+        user_allow_update: False