Add support for Keystone's fernet tokens

Keystone supports (and that's a default setting since Ocata) using
non-persistent fernet tokens instead of UUID tokens written into the DB.
This setting is in some cases better in terms of performance and
manageability (no more tokens DB table cleanups). OpenStack-Helm should
be able to support it.

General issue with fernet tokens is that keys used to encrypt them need
to be persistent and shared accross the cluster. Moreover "rotate"
operation generates a new key, so key repository will change over time.

This commit implements fernet tokens support by:
* A 'keystone-fernet-keys' secret is created to serve as keys repository.
* New fernet-setup Job will populate secret with initial keys.
* New fernet-rotate CronJob will be run periodically (weekly by default)
  and perform key rotation operation and update the secret.
* Secret is attached to keystone-api pods in /etc/keystone/fernet-tokens
  directory.

Turns out k8s is updating secrets attached to pods automatically, so
because of Keystone's fernet tokens implementation, we don't need to
worry about synchronization of the key repository. Everything should be
fine unless fernet-rotate job will run before all of the pods will
notice the change in the secret. As in real-world scenario you would
rotate your keys no more often than once an hour, this should be totally
fine.

Implements: blueprint keystone-fernet-tokens
Change-Id: Ifc84b8c97e1a85d30eb46260582d9c58220fbf0a

doc/source/install/multinode.rst

@@ -174,6 +174,19 @@ component so we relax the access control rules:
 
     kubectl update -f https://raw.githubusercontent.com/openstack/openstack-helm/master/tools/kubeadm-aio/assets/opt/rbac/dev.yaml
 
+Enabling Cron Jobs
+------------------
+
+OpenStack-Helm's default Keystone token provider is `fernet
+<https://docs.openstack.org/keystone/latest/admin/identity-fernet-token-faq.html>`_.
+To provide sufficient security, keys used to generate fernet tokens need to be
+rotated regularly. Keystone chart provides Cron Job for that task, but it is
+only deployed when Cron Jobs API is enabled on Kubernetes cluster. To enable
+Cron Jobs add ``--runtime-config=batch/v2alpha1=true`` to your kube-apiserver
+startup arguments (e.g. in your
+``/etc/kubernetes/manifests/kube-apiserver.yaml`` manifest). By default fernet
+keys will be rotated weekly.
+
 Preparing Persistent Storage
 ----------------------------
 

keystone/templates/bin/_fernet-manage.py.tpl

@@ -0,0 +1,162 @@
+#!/usr/bin/env python
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import base64
+import errno
+import grp
+import logging
+import os
+import pwd
+import re
+import six
+import subprocess
+import sys
+
+import requests
+
+FERNET_DIR = os.environ['KEYSTONE_KEYS_REPOSITORY']
+KEYSTONE_USER = os.environ['KEYSTONE_USER']
+KEYSTONE_GROUP = os.environ['KEYSTONE_GROUP']
+SECRET_NAME = 'keystone-fernet-keys'
+NAMESPACE = os.environ['KUBERNETES_NAMESPACE']
+
+# k8s connection data
+KUBE_HOST = None
+KUBE_CERT = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'
+KUBE_TOKEN = None
+
+LOG_DATEFMT = "%Y-%m-%d %H:%M:%S"
+LOG_FORMAT = "%(asctime)s.%(msecs)03d - %(levelname)s - %(message)s"
+logging.basicConfig(format=LOG_FORMAT, datefmt=LOG_DATEFMT)
+LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.INFO)
+
+
+def read_kube_config():
+    global KUBE_HOST, KUBE_TOKEN
+    KUBE_HOST = "https://%s:%s" % ('kubernetes.default',
+                                   os.environ['KUBERNETES_SERVICE_PORT'])
+    with open('/var/run/secrets/kubernetes.io/serviceaccount/token', 'r') as f:
+        KUBE_TOKEN = f.read()
+
+
+def get_secret_definition(name):
+    url = '%s/api/v1/namespaces/%s/secrets/%s' % (KUBE_HOST, NAMESPACE, name)
+    resp = requests.get(url,
+                        headers={'Authorization': 'Bearer %s' % KUBE_TOKEN},
+                        verify=KUBE_CERT)
+    if resp.status_code != 200:
+        LOG.error('Cannot get secret %s.', name)
+        LOG.error(resp.text)
+        return None
+    return resp.json()
+
+
+def update_secret(name, secret):
+    url = '%s/api/v1/namespaces/%s/secrets/%s' % (KUBE_HOST, NAMESPACE, name)
+    resp = requests.put(url,
+                        json=secret,
+                        headers={'Authorization': 'Bearer %s' % KUBE_TOKEN},
+                        verify=KUBE_CERT)
+    if resp.status_code != 200:
+        LOG.error('Cannot update secret %s.', name)
+        LOG.error(resp.text)
+        return False
+    return True
+
+
+def read_from_files():
+    keys = filter(
+        lambda name: os.path.isfile(FERNET_DIR + name) and re.match("^\d+$",
+                                                                    name),
+        os.listdir(FERNET_DIR)
+    )
+    data = {}
+    for key in keys:
+        with open(FERNET_DIR + key, 'r') as f:
+            data[key] = f.read()
+    if len(keys):
+        LOG.debug("Keys read from files: %s", keys)
+    else:
+        LOG.warn("No keys were read from files.")
+    return data
+
+
+def get_keys_data():
+    keys = read_from_files()
+    return dict([(key, base64.b64encode(value.encode()).decode())
+                for (key, value) in six.iteritems(keys)])
+
+
+def write_to_files(data):
+    if not os.path.exists(os.path.dirname(FERNET_DIR)):
+        try:
+            os.makedirs(os.path.dirname(FERNET_DIR))
+        except OSError as exc: # Guard against race condition
+            if exc.errno != errno.EEXIST:
+                raise
+        uid = pwd.getpwnam(KEYSTONE_USER).pw_uid
+        gid = grp.getgrnam(KEYSTONE_GROUP).gr_gid
+        os.chown(FERNET_DIR, uid, gid)
+
+    for (key, value) in six.iteritems(data):
+        with open(FERNET_DIR + key, 'w') as f:
+            decoded_value = base64.b64decode(value).decode()
+            f.write(decoded_value)
+            LOG.debug("Key %s: %s", key, decoded_value)
+    LOG.info("%s keys were written", len(data))
+
+
+def execute_command(cmd):
+    LOG.info("Executing 'keystone-manage %s --keystone-user=%s "
+             "--keystone-group=%s' command.",
+             cmd, KEYSTONE_USER, KEYSTONE_GROUP)
+    subprocess.call(['keystone-manage', cmd,
+                     '--keystone-user=%s' % KEYSTONE_USER,
+                     '--keystone-group=%s' % KEYSTONE_GROUP])
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('command', choices=['fernet_setup', 'fernet_rotate'])
+    args = parser.parse_args()
+
+    read_kube_config()
+    secret = get_secret_definition(SECRET_NAME)
+    if not secret:
+        LOG.error("Secret '%s' does not exist.", SECRET_NAME)
+        sys.exit(1)
+
+    if args.command == 'fernet_rotate':
+        LOG.info("Copying existing fernet keys from secret '%s' to %s.",
+                 SECRET_NAME, FERNET_DIR)
+        write_to_files(secret['data'])
+
+    execute_command(args.command)
+
+    LOG.info("Updating data for '%s' secret.", SECRET_NAME)
+    updated_keys = get_keys_data()
+    secret['data'] = updated_keys
+    if not update_secret(SECRET_NAME, secret):
+        sys.exit(1)
+    LOG.info("%s fernet keys have been placed to secret '%s'",
+             len(updated_keys), SECRET_NAME)
+    LOG.debug("Placed keys: %s", updated_keys)
+    LOG.info("Fernet keys %s has been completed",
+             "rotation" if args.command == 'fernet_rotate' else "generation")
+
+if __name__ == "__main__":
+    main()

keystone/templates/configmap-bin.yaml

@@ -31,3 +31,5 @@ data:
 {{ tuple "bin/_db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   keystone-api.sh: |
 {{ tuple "bin/_keystone-api.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  fernet-manage.py: |
+{{ tuple "bin/_fernet-manage.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

keystone/templates/cron-job-fernet-rotate.yaml

@@ -0,0 +1,76 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if and (eq .Values.conf.keystone.token.keystone.provider "fernet") (.Capabilities.APIVersions.Has "batch/v2alpha1") }}
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.fernet_rotate }}
+{{- $mounts_keystone_fernet_rotate := .Values.pod.mounts.keystone_fernet_rotate.keystone_fernet_rotate }}
+{{- $mounts_keystone_fernet_rotate_init := .Values.pod.mounts.keystone_fernet_rotate.init_container }}
+apiVersion: batch/v2alpha1
+kind: CronJob
+metadata:
+  name: keystone-fernet-rotate
+spec:
+  schedule: {{ .Values.jobs.fernet_rotate.cron | quote }}
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          initContainers:
+{{ tuple $envAll $dependencies $mounts_keystone_fernet_rotate_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }}
+          restartPolicy: OnFailure
+          nodeSelector:
+            {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+          containers:
+            - name: keystone-fernet-rotate
+              image: {{ .Values.images.fernet_rotate }}
+              imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.fernet_rotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
+              env:
+              - name: KEYSTONE_USER
+                value: {{ .Values.jobs.fernet_rotate.user | quote }}
+              - name: KEYSTONE_GROUP
+                value: {{ .Values.jobs.fernet_rotate.group | quote }}
+              - name: KUBERNETES_NAMESPACE
+                value: {{ .Release.Namespace | quote }}
+              - name: KEYSTONE_KEYS_REPOSITORY
+                value: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository | quote }}
+              command:
+                - python
+                - /tmp/fernet-manage.py
+                - fernet_rotate
+              volumeMounts:
+              - name: etckeystone
+                mountPath: /etc/keystone
+              - name: keystone-etc
+                mountPath: /etc/keystone/keystone.conf
+                subPath: keystone.conf
+                readOnly: true
+              - name: keystone-bin
+                mountPath: /tmp/fernet-manage.py
+                subPath: fernet-manage.py
+                readOnly: true
+    {{- if $mounts_keystone_fernet_rotate.volumeMounts }}{{ toYaml $mounts_keystone_fernet_rotate.volumeMounts | indent 14 }}{{ end }}
+          volumes:
+          - name: etckeystone
+            emptyDir: {}
+          - name: keystone-etc
+            configMap:
+              name: keystone-etc
+          - name: keystone-bin
+            configMap:
+              name: keystone-bin
+    {{- if $mounts_keystone_fernet_rotate.volumes }}{{ toYaml $mounts_keystone_fernet_rotate.volumes | indent 10 }}{{ end }}
+{{- end }}

keystone/templates/deployment-api.yaml

@@ -94,6 +94,10 @@ spec:
             mountPath: /tmp/keystone-api.sh
             subPath: keystone-api.sh
             readOnly: true
+{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }}
+          - name: keystone-fernet-keys
+            mountPath: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository }}
+{{- end }}
 {{- if $mounts_keystone_api.volumeMounts }}{{ toYaml $mounts_keystone_api.volumeMounts | indent 10 }}{{ end }}
       volumes:
       - name: etckeystone
@@ -108,4 +112,9 @@ spec:
         configMap:
           name: keystone-bin
           defaultMode: 0555
+{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }}
+      - name: keystone-fernet-keys
+        secret:
+          secretName: keystone-fernet-keys
+{{- end }}
 {{- if $mounts_keystone_api.volumes }}{{ toYaml $mounts_keystone_api.volumes | indent 6 }}{{ end }}

keystone/templates/job-db-sync.yaml

@@ -59,6 +59,10 @@ spec:
             mountPath: /tmp/db-sync.sh
             subPath: db-sync.sh
             readOnly: true
+{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }}
+          - name: keystone-fernet-keys
+            mountPath: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository }}
+{{- end }}
 {{- if $mounts_keystone_db_sync.volumeMounts }}{{ toYaml $mounts_keystone_db_sync.volumeMounts | indent 10 }}{{ end }}
       volumes:
       - name: etckeystone
@@ -71,4 +75,9 @@ spec:
         configMap:
           name: keystone-bin
           defaultMode: 0555
+{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }}
+      - name: keystone-fernet-keys
+        secret:
+          secretName: keystone-fernet-keys
+{{- end }}
 {{- if $mounts_keystone_db_sync.volumes }}{{ toYaml $mounts_keystone_db_sync.volumes | indent 6 }}{{ end }}

keystone/templates/job-fernet-setup.yaml

@@ -0,0 +1,72 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }}
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.fernet_setup }}
+{{- $mounts_keystone_fernet_setup := .Values.pod.mounts.keystone_fernet_setup.keystone_fernet_setup }}
+{{- $mounts_keystone_fernet_setup_init := .Values.pod.mounts.keystone_fernet_setup.init_container }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: keystone-fernet-setup
+spec:
+  template:
+    spec:
+      initContainers:
+{{ tuple $envAll $dependencies $mounts_keystone_fernet_setup_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+      containers:
+        - name: keystone-fernet-setup
+          image: {{ .Values.images.fernet_setup }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.fernet_setup | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+          - name: KEYSTONE_USER
+            value: {{ .Values.jobs.fernet_setup.user | quote }}
+          - name: KEYSTONE_GROUP
+            value: {{ .Values.jobs.fernet_setup.group | quote }}
+          - name: KUBERNETES_NAMESPACE
+            value: {{ .Release.Namespace | quote }}
+          - name: KEYSTONE_KEYS_REPOSITORY
+            value: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository | quote }}
+          command:
+            - python
+            - /tmp/fernet-manage.py
+            - fernet_setup
+          volumeMounts:
+          - name: etckeystone
+            mountPath: /etc/keystone
+          - name: keystone-etc
+            mountPath: /etc/keystone/keystone.conf
+            subPath: keystone.conf
+            readOnly: true
+          - name: keystone-bin
+            mountPath: /tmp/fernet-manage.py
+            subPath: fernet-manage.py
+            readOnly: true
+{{- if $mounts_keystone_fernet_setup.volumeMounts }}{{ toYaml $mounts_keystone_fernet_setup.volumeMounts | indent 10 }}{{ end }}
+      volumes:
+      - name: etckeystone
+        emptyDir: {}
+      - name: keystone-etc
+        configMap:
+          name: keystone-etc
+      - name: keystone-bin
+        configMap:
+          name: keystone-bin
+{{- if $mounts_keystone_fernet_setup.volumes }}{{ toYaml $mounts_keystone_fernet_setup.volumes | indent 6 }}{{ end }}
+{{- end }}

keystone/templates/secret-fernet-keys.yaml

@@ -0,0 +1,22 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: keystone-fernet-keys
+type: Opaque
+data:
+{{- end }}

keystone/values.yaml

@@ -26,6 +26,8 @@ images:
   test: docker.io/kolla/ubuntu-binary-rally:4.0.0
   db_init: docker.io/kolla/ubuntu-source-keystone:3.0.3
   db_sync: docker.io/kolla/ubuntu-source-keystone:3.0.3
+  fernet_setup: docker.io/kolla/ubuntu-source-keystone:3.0.3
+  fernet_rotate: docker.io/kolla/ubuntu-source-keystone:3.0.3
   api: docker.io/kolla/ubuntu-source-keystone:3.0.3
   dep_check: docker.io/kolla/ubuntu-source-kubernetes-entrypoint:4.0.0
   pull_policy: "IfNotPresent"
@@ -58,6 +60,8 @@ dependencies:
   api:
     jobs:
     - keystone-db-sync
+    # Comment line below when not running fernet tokens.
+    - keystone-fernet-setup
     services:
     - service: oslo_cache
       endpoint: internal
@@ -70,9 +74,15 @@ dependencies:
   db_sync:
     jobs:
     - keystone-db-init
+    # Comment line below when not running fernet tokens.
+    - keystone-fernet-setup
     services:
     - service: oslo_db
       endpoint: internal
+  fernet_setup:
+  fernet_rotate:
+    jobs:
+    - keystone-fernet-setup
   tests:
     services:
     - service: identity
@@ -105,6 +115,12 @@ pod:
     keystone_bootstrap:
       init_container: null
       keystone_bootstrap:
+    keystone_fernet_setup:
+      init_container: null
+      keystone_fernet_setup:
+    keystone_fernet_rotate:
+      init_container: null
+      keystone_fernet_rotate:
   replicas:
     api: 1
   lifecycle:
@@ -159,6 +175,30 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+      fernet_setup:
+        limits:
+          memory: "128Mi"
+          cpu: "100m"
+        requests:
+          memory: "10024Mi"
+          cpu: "2000m"
+      fernet_rotate:
+        limits:
+          memory: "128Mi"
+          cpu: "100m"
+        requests:
+          memory: "1024Mi"
+          cpu: "2000m"
+
+jobs:
+  fernet_setup:
+    user: keystone
+    group: keystone
+  fernet_rotate:
+    # weekly
+    cron: "0 0 * * 0"
+    user: keystone
+    group: keystone
 
 conf:
   rally_tests:
@@ -175,10 +215,13 @@ conf:
     append:
     default:
       keystone:
-        max_token_size: 32
+        max_token_size: 255
     token:
       keystone:
-        provider: uuid
+        provider: fernet
+    fernet_tokens:
+      keystone:
+        key_repository: /etc/keystone/fernet-keys/
     database:
       oslo:
         db:

tools/kubeadm-aio/assets/etc/kubeadm.conf

@@ -0,0 +1,4 @@
+apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+apiServerExtraArgs:
+  runtime-config: "batch/v2alpha1=true"

tools/kubeadm-aio/assets/usr/bin/kubeadm-aio

@@ -31,9 +31,9 @@ if [[ "${KUBE_ROLE}" == "master" ]]; then
   if [[ "$KUBE_BIND_DEV" != "autodetect" ]]; then
     KUBE_BIND_IP=$(ip addr list ${KUBE_BIND_DEV} |grep "inet " |cut -d' ' -f6|cut -d/ -f1)
     echo 'We are going to bind the K8s API to: ${KUBE_BIND_IP}'
-    kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} --api-advertise-addresses ${KUBE_BIND_IP}
+    kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} --api-advertise-addresses ${KUBE_BIND_IP} --config /etc/kubeadm.conf
   else
-    kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG}
+    kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} --config /etc/kubeadm.conf
   fi
 
   echo 'Setting up K8s client'