KubeADM: Keystone Kubernetes Webhook

This PS adds the ability to deploy the Keystone Kubernetes Webhook
chart via kubeadm-aio

Change-Id: I18b0477a775de942f940e9c0984559089dca1cdb
Co-Authored-By: Tin Lam <tin@irrational.io>
Co-Authored-By: Gage Hugo <gagehugo@gmail.com>
Signed-off-by: Pete Birley <pete@port.direct>

.zuul.yaml

@@ -41,6 +41,12 @@
               - ^.*\.rst$
               - ^doc/.*$
               - ^releasenotes/.*$
+        - openstack-helm-infra-kubernetes-keystone-auth:
+            voting: false
+            irrelevant-files:
+              - ^.*\.rst$
+              - ^doc/.*$
+              - ^releasenotes/.*$
     gate:
       jobs:
         - openstack-helm-infra-linter:
@@ -245,3 +251,14 @@
     name: openstack-helm-infra-five-fedora
     parent: openstack-helm-infra
     nodeset: openstack-helm-five-node-fedora
+
+- job:
+    name: openstack-helm-infra-kubernetes-keystone-auth
+    vars:
+      zuul_osh_relative_path: ../openstack-helm/
+      kubernetes_keystone_auth: true
+    parent: openstack-helm-infra
+    nodeset: openstack-helm-single-node
+    run: playbooks/osh-infra-keystone-k8s-auth.yaml
+    required-projects:
+      - openstack/openstack-helm

kubernetes-keystone-webhook/values.yaml

@@ -116,6 +116,7 @@ endpoints:
           key: null
   identity:
     name: keystone
+    namespace: null
     auth:
       admin:
         region_name: RegionOne

playbooks/osh-infra-keystone-k8s-auth.yaml

@@ -0,0 +1,93 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Setup OS and K8s Clients
+      shell: |
+        set -xe;
+        cd "${OSH_PATH}"
+        ./tools/deployment/developer/nfs/020-setup-client.sh
+      environment:
+        OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+        OSH_PATH: "{{ zuul_osh_relative_path | default('') }}"
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy Ingress
+      shell: |
+        set -xe;
+        cd "${OSH_PATH}"
+        ./tools/deployment/developer/nfs/030-ingress.sh
+      environment:
+        OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+        OSH_PATH: "{{ zuul_osh_relative_path | default('') }}"
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy NFS
+      shell: |
+        set -xe;
+        cd "${OSH_PATH}"
+        ./tools/deployment/developer/nfs/040-nfs-provisioner.sh
+      environment:
+        OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+        OSH_PATH: "{{ zuul_osh_relative_path | default('') }}"
+        OSH_INFRA_PATH: "../openstack-helm-infra/"
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy Mariadb
+      shell: |
+        set -xe;
+        cd "${OSH_PATH}"
+        ./tools/deployment/developer/nfs/050-mariadb.sh
+      environment:
+        OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+        OSH_PATH: "{{ zuul_osh_relative_path | default('') }}"
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy RabbitMQ
+      shell: |
+        set -xe;
+        cd "${OSH_PATH}"
+        ./tools/deployment/developer/nfs/060-rabbitmq.sh
+      environment:
+        OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+        OSH_PATH: "{{ zuul_osh_relative_path | default('') }}"
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy Memcached
+      shell: |
+        set -xe;
+        cd "${OSH_PATH}"
+        ./tools/deployment/developer/nfs/070-memcached.sh
+      environment:
+        OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+        OSH_PATH: "{{ zuul_osh_relative_path | default('') }}"
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy Keystone
+      shell: |
+        set -xe;
+        cd "${OSH_PATH}"
+        ./tools/deployment/developer/nfs/080-keystone.sh
+      environment:
+        OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+        OSH_PATH: "{{ zuul_osh_relative_path | default('') }}"
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Check Kubernetes Keystone Auth
+      shell: |
+        set -xe;
+        ./tools/deployment/keystone-auth/check.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"

roles/build-images/tasks/kubeadm-aio.yaml

@@ -52,7 +52,7 @@
                 --build-arg KUBE_VERSION="{{ version.kubernetes }}" \
                 --build-arg CNI_VERSION="{{ version.cni }}" \
                 --build-arg HELM_VERSION="{{ version.helm }}" \
-                --build-arg CHARTS="calico,flannel,tiller,kube-dns" \
+                --build-arg CHARTS="calico,flannel,tiller,kube-dns,kubernetes-keystone-webhook" \
                 --build-arg HTTP_PROXY="{{ proxy.http }}" \
                 --build-arg HTTPS_PROXY="{{ proxy.https }}" \
                 --build-arg NO_PROXY="{{ proxy.noproxy }}" \
@@ -78,7 +78,7 @@
                 --build-arg KUBE_VERSION="{{ version.kubernetes }}" \
                 --build-arg CNI_VERSION="{{ version.cni }}" \
                 --build-arg HELM_VERSION="{{ version.helm }}" \
-                --build-arg CHARTS="calico,flannel,tiller,kube-dns" \
+                --build-arg CHARTS="calico,flannel,tiller,kube-dns,kubernetes-keystone-webhook" \
                 {% if zuul_site_mirror_fqdn is defined and zuul_site_mirror_fqdn %}
                 --build-arg UBUNTU_URL="http://{{ zuul_site_mirror_fqdn }}/ubuntu/" \
                 --build-arg ALLOW_UNAUTHENTICATED="true" \

roles/deploy-kubeadm-aio-common/defaults/main.yml

@@ -17,6 +17,7 @@ kubernetes_cluster_pod_subnet: 192.168.0.0/16
 kubernetes_cluster_domain: cluster.local
 kubernetes_network_default_device: null
 kubernetes_selfhosted: false
+kubernetes_keystone_auth: false
 
 images:
   kubernetes:

roles/deploy-kubeadm-aio-common/tasks/util-kubeadm-aio-run.yaml

@@ -51,6 +51,7 @@
           CONTAINER_RUNTIME=docker
           KUBELET_NODE_LABELS="{{ kubeadm_kubelet_labels }}"
           KUBE_SELF_HOSTED="{{ kubernetes_selfhosted }}"
+          KUBE_KEYSTONE_AUTH="{{ kubernetes_keystone_auth }}"
       register: kubeadm_master_deploy
   rescue:
     - name: "getting logs for {{ kubeadm_aio_action }} action"

tools/deployment/keystone-auth/check.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+export OS_CLOUD=openstack_helm
+function keystone_token () {
+  openstack token issue -f value -c id
+}
+sudo cp -va $HOME/.kube/config /tmp/kubeconfig.yaml
+sudo kubectl --kubeconfig /tmp/kubeconfig.yaml config unset users.kubernetes-admin
+
+# Test
+if ! kubectl --kubeconfig /tmp/kubeconfig.yaml --token "$(keystone_token)" get pods ; then
+  echo "Denied, as expected by policy"
+else
+  exit 1
+fi
+kubectl --kubeconfig /tmp/kubeconfig.yaml --token "$(keystone_token)" get pods -n openstack
+
+# create a demoUser
+openstack user create --or-show --password demoPassword demoUser
+unset OS_CLOUD
+export OS_AUTH_URL="http://keystone.openstack.svc.cluster.local/v3"
+export OS_IDENTITY_API_VERSION="3"
+export OS_PASSWORD="demoPassword"
+export OS_USERNAME="demoUser"
+
+# See this does fail as the policy does not allow for a non-admin user
+TOKEN=$(openstack token issue -f value -c id)
+if ! kubectl --kubeconfig /tmp/kubeconfig.yaml --token "$(keystone_token)" get pods -n openstack ; then
+  echo "Denied, as expected by policy"
+else
+  exit 1
+fi

tools/images/kubeadm-aio/Dockerfile

@@ -37,7 +37,7 @@ ENV CNI_VERSION ${CNI_VERSION}
 ARG HELM_VERSION="v2.8.2"
 ENV HELM_VERSION ${HELM_VERSION}
 
-ARG CHARTS="calico,flannel,tiller,kube-dns"
+ARG CHARTS="calico,flannel,tiller,kube-dns,kubernetes-keystone-webhook"
 ENV CHARTS ${CHARTS}
 
 ARG HTTP_PROXY=""

tools/images/kubeadm-aio/assets/entrypoint.sh

@@ -52,6 +52,7 @@ fi
 : ${KUBE_API_BIND_ADDR:="${KUBE_BIND_ADDR}"}
 : ${KUBE_CERTS_DIR:="/etc/kubernetes/pki"}
 : ${KUBE_SELF_HOSTED:="false"}
+: ${KUBE_KEYSTONE_AUTH:="false"}
 : ${KUBELET_NODE_LABELS:=""}
 
 PLAYBOOK_VARS="{
@@ -78,6 +79,7 @@ PLAYBOOK_VARS="{
     \"imageRepository\": \"${KUBE_IMAGE_REPO}\",
     \"certificatesDir\": \"${KUBE_CERTS_DIR}\",
     \"selfHosted\": \"${KUBE_SELF_HOSTED}\",
+    \"keystoneAuth\": \"${KUBE_KEYSTONE_AUTH}\",
     \"api\": {
       \"bindPort\": ${KUBE_API_BIND_PORT}
     },

tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/helm-keystone-auth.yaml

@@ -0,0 +1,101 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+- name: setting up bootstrap tiller
+  block:
+    - name: pull the helm tiller Image
+      become: true
+      become_user: root
+      docker_image:
+        pull: true
+        name: "{{ helm.tiller_image }}"
+    - name: deploying bootstrap tiller
+      become: true
+      become_user: root
+      docker_container:
+        name: "helm-tiller"
+        image: "{{ helm.tiller_image }}"
+        state: started
+        detach: true
+        recreate: yes
+        network_mode: host
+        volumes:
+          - /etc/kubernetes/admin.conf:/etc/kubernetes/admin.conf:ro
+        env:
+          KUBECONFIG=/etc/kubernetes/admin.conf
+      register: kubeadm_aio_tiller_container
+      ignore_errors: True
+    - name: wait for tiller to be ready
+      delegate_to: 127.0.0.1
+      command: helm version --server
+      environment:
+        HELM_HOST: 'localhost:44134'
+      register: task_result
+      until: task_result.rc == 0
+      retries: 120
+      delay: 5
+
+- name: kubeadm | get certs
+  block:
+    - name: kubeadm | get kubeapi cert
+      shell: cat /etc/kubernetes/pki/apiserver.crt | base64 -w0
+      register: kubeadm_kubeapi_cert
+    - name: kubeadm | get kubeapi key
+      shell: cat /etc/kubernetes/pki/apiserver.key | base64 -w0
+      register: kubeadm_kubeapi_key
+
+- name: kubeadm | keystone auth
+  delegate_to: 127.0.0.1
+  block:
+    - name: kubeadm | keystone auth
+      command: "helm upgrade --install kubernetes-keystone-webhook /opt/charts/kubernetes-keystone-webhook --namespace=kube-system --set endpoints.identity.namespace=openstack --set endpoints.kubernetes.auth.api.tls.crt={{ kubeadm_kubeapi_cert.stdout }} --set endpoints.kubernetes.auth.api.tls.key={{ kubeadm_kubeapi_key.stdout }}"
+      environment:
+        HELM_HOST: 'localhost:44134'
+    - name: kubeadm | keystone auth
+      command: helm status kubernetes-keystone-webhook
+      environment:
+        HELM_HOST: 'localhost:44134'
+      register: kubeadm_helm_keystone_status
+    - name: kubeadm | keystone auth
+      debug:
+        msg: "{{ kubeadm_helm_keystone_status }}"
+
+- name: kubeadm | setup api server for keystone
+  block:
+    - name: kubeadm | copying webhook config to host
+      become: true
+      become_user: root
+      template:
+        src: webhook.kubeconfig.j2
+        dest: /etc/kubernetes/pki/webhook.kubeconfig
+        mode: 0640
+    - name: kubeadm | configuring api server
+      become: true
+      become_user: root
+      shell: |
+        # TODO(lamt): Clean up this way of restarting the kube-apiserver. Preferably,
+        # the setting is in place when the kube-apiserver comes up. Currently, the
+        # kube-apiserver does not start whenever the webhook fails.
+        cat /etc/kubernetes/manifests/kube-apiserver.yaml > /tmp/kube-apiserver.yaml
+        sed -i '/etcd-keyfile/a \ \ \ \ -\ --authentication-token-webhook-config-file=/etc/kubernetes/pki/webhook.kubeconfig\n \ \ \ \- --authorization-webhook-config-file=/etc/kubernetes/pki/webhook.kubeconfig' /tmp/kube-apiserver.yaml
+        sed -i -e 's/Node,RBAC/Node,Webhook,RBAC/g' /tmp/kube-apiserver.yaml
+        sed -i '/hostNetwork: true/a\ \ dnsPolicy: ClusterFirstWithHostNet' /tmp/kube-apiserver.yaml
+        mv /tmp/kube-apiserver.yaml /etc/kubernetes/manifests/kube-apiserver.yaml
+
+- name: "removing bootstrap tiller container"
+  become: true
+  become_user: root
+  docker_container:
+    name: "helm-tiller"
+    state: absent

tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/tasks/main.yaml

@@ -136,14 +136,9 @@
         executable: /bin/bash
     - include_tasks: wait-for-kube-system-namespace.yaml
 
-# - name: deploying kube-dns addon
-#   delegate_to: 127.0.0.1
-#   block:
-#     - name: master | deploy | kube-dns
-#       command: kubeadm alpha phase addon kube-dns --config /mnt/rootfs/etc/kubernetes/kubeadm-conf.yaml
-#     - include_tasks: wait-for-kube-system-namespace.yaml
-
 - include_tasks: helm-dns.yaml
+- include_tasks: helm-keystone-auth.yaml
+  when: k8s.keystoneAuth
 - include_tasks: helm-deploy.yaml
 
 - name: uploading cluster config to api

tools/images/kubeadm-aio/assets/opt/playbooks/roles/deploy-kubeadm-master/templates/webhook.kubeconfig.j2

@@ -0,0 +1,16 @@
+apiVersion: v1
+clusters:
+  - cluster:
+      insecure-skip-tls-verify: true
+      server: https://k8sksauth-api.kube-system.svc.cluster.local:8443/webhook
+    name: webhook
+contexts:
+  - context:
+      cluster: webhook
+      user: webhook
+    name: webhook
+current-context: webhook
+kind: Config
+preferences: {}
+users:
+  - name: webhook

tools/images/kubeadm-aio/assets/opt/playbooks/vars.yaml

@@ -37,6 +37,7 @@ all:
       imageRepository: gcr.io/google_containers
       certificatesDir: /etc/kubernetes/pki
       selfHosted: false
+      keystoneAuth: false
       api:
         bindPort: 6443
         #NOTE(portdirect): The following is a custom key, which resolves the