From cfab320f260b698cbf388254cd762b603707e5bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Dulko?= Date: Mon, 29 May 2017 14:07:29 +0200 Subject: [PATCH] Add setup and rotate job for credential keys Keystone is using keys to encrypt credentials saved into the database. The mechanism is very similar to fernet tokens. This commit implements a job setting key repository up and rotate job for those keys. All is based on implementation of fernet tokens. Change-Id: I88faf1d02d2b317563e8603cebba542f8b133c6a Closes-Bug: 1693807 --- doc/source/install/multinode.rst | 4 + keystone/templates/bin/_fernet-manage.py.tpl | 36 +++++++-- .../templates/cron-job-credential-rotate.yaml | 78 +++++++++++++++++++ keystone/templates/deployment-api.yaml | 5 ++ keystone/templates/job-credential-setup.yaml | 70 +++++++++++++++++ .../templates/secret-credential-keys.yaml | 20 +++++ keystone/values.yaml | 40 ++++++++++ 7 files changed, 246 insertions(+), 7 deletions(-) create mode 100644 keystone/templates/cron-job-credential-rotate.yaml create mode 100644 keystone/templates/job-credential-setup.yaml create mode 100644 keystone/templates/secret-credential-keys.yaml diff --git a/doc/source/install/multinode.rst b/doc/source/install/multinode.rst index 088b490a9d..417666b6fe 100644 --- a/doc/source/install/multinode.rst +++ b/doc/source/install/multinode.rst @@ -187,6 +187,10 @@ startup arguments (e.g. in your ``/etc/kubernetes/manifests/kube-apiserver.yaml`` manifest). By default fernet keys will be rotated weekly. +Please note that similar solution is used for keys used to encrypt credentials +saved by Keystone. Those keys are also rotated by another Cron Job. By default +it is run in a monthly manner. + Preparing Persistent Storage ---------------------------- diff --git a/keystone/templates/bin/_fernet-manage.py.tpl b/keystone/templates/bin/_fernet-manage.py.tpl index 4b484fae1b..30f1ab54fd 100644 --- a/keystone/templates/bin/_fernet-manage.py.tpl +++ b/keystone/templates/bin/_fernet-manage.py.tpl @@ -25,13 +25,13 @@ import re import six import subprocess import sys +import time import requests FERNET_DIR = os.environ['KEYSTONE_KEYS_REPOSITORY'] KEYSTONE_USER = os.environ['KEYSTONE_USER'] KEYSTONE_GROUP = os.environ['KEYSTONE_GROUP'] -SECRET_NAME = 'keystone-fernet-keys' NAMESPACE = os.environ['KUBERNETES_NAMESPACE'] # k8s connection data @@ -131,18 +131,26 @@ def execute_command(cmd): def main(): parser = argparse.ArgumentParser() - parser.add_argument('command', choices=['fernet_setup', 'fernet_rotate']) + parser.add_argument('command', choices=['fernet_setup', 'fernet_rotate', + 'credential_setup', + 'credential_rotate']) args = parser.parse_args() + is_credential = args.command.startswith('credential') + + SECRET_NAME = ('keystone-credential-keys' if is_credential else + 'keystone-fernet-keys') + read_kube_config() secret = get_secret_definition(SECRET_NAME) if not secret: LOG.error("Secret '%s' does not exist.", SECRET_NAME) sys.exit(1) - if args.command == 'fernet_rotate': - LOG.info("Copying existing fernet keys from secret '%s' to %s.", - SECRET_NAME, FERNET_DIR) + if args.command in ('fernet_rotate', 'credential_rotate'): + LOG.info("Copying existing %s keys from secret '%s' to %s.", + 'credential' if is_credential else 'fernet', SECRET_NAME, + FERNET_DIR) write_to_files(secret['data']) execute_command(args.command) @@ -155,8 +163,22 @@ def main(): LOG.info("%s fernet keys have been placed to secret '%s'", len(updated_keys), SECRET_NAME) LOG.debug("Placed keys: %s", updated_keys) - LOG.info("Fernet keys %s has been completed", - "rotation" if args.command == 'fernet_rotate' else "generation") + LOG.info("%s keys %s has been completed", + "Credential" if is_credential else 'Fernet', + "rotation" if args.command.endswith('_rotate') else "generation") + + if args.command == 'credential_rotate': + # `credential_rotate` needs doing `credential_migrate` as well once all + # of the nodes have the new keys. So we'll sleep configurable amount of + # time to make sure k8s reloads the secrets in all pods and then + # execute `credential_migrate`. + + migrate_wait = os.environ['KEYSTONE_CREDENTIAL_MIGRATE_WAIT'] + LOG.info("Waiting %d seconds to execute `credential_migrate`.", + migrate_wait) + time.sleep(migrate_wait) + + execute_command('credential_migrate') if __name__ == "__main__": main() diff --git a/keystone/templates/cron-job-credential-rotate.yaml b/keystone/templates/cron-job-credential-rotate.yaml new file mode 100644 index 0000000000..e4dc1bc703 --- /dev/null +++ b/keystone/templates/cron-job-credential-rotate.yaml @@ -0,0 +1,78 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Capabilities.APIVersions.Has "batch/v2alpha1"}} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.credential_rotate }} +{{- $mounts_keystone_credential_rotate := .Values.pod.mounts.keystone_credential_rotate.keystone_credential_rotate }} +{{- $mounts_keystone_credential_rotate_init := .Values.pod.mounts.keystone_credential_rotate.init_container }} +apiVersion: batch/v2alpha1 +kind: CronJob +metadata: + name: keystone-credential-rotate +spec: + schedule: {{ .Values.jobs.credential_rotate.cron | quote }} + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + initContainers: +{{ tuple $envAll $dependencies $mounts_keystone_credential_rotate_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + containers: + - name: keystone-credential-rotate + image: {{ .Values.images.credential_rotate }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.credential_rotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} + env: + - name: KEYSTONE_USER + value: {{ .Values.jobs.credential_rotate.user | quote }} + - name: KEYSTONE_GROUP + value: {{ .Values.jobs.credential_rotate.group | quote }} + - name: KUBERNETES_NAMESPACE + value: {{ .Release.Namespace | quote }} + - name: KEYSTONE_KEYS_REPOSITORY + value: {{ .Values.conf.keystone.credential.keystone.key_repository | quote }} + - name: KEYSTONE_CREDENTIAL_MIGRATE_WAIT + value: {{ .Values.jobs.credential_rotate.migrate_wait | quote }} + command: + - python + - /tmp/fernet-manage.py + - credential_rotate + volumeMounts: + - name: etckeystone + mountPath: /etc/keystone + - name: keystone-etc + mountPath: /etc/keystone/keystone.conf + subPath: keystone.conf + readOnly: true + - name: keystone-bin + mountPath: /tmp/fernet-manage.py + subPath: fernet-manage.py + readOnly: true + {{- if $mounts_keystone_credential_rotate.volumeMounts }}{{ toYaml $mounts_keystone_credential_rotate.volumeMounts | indent 14 }}{{ end }} + volumes: + - name: etckeystone + emptyDir: {} + - name: keystone-etc + configMap: + name: keystone-etc + - name: keystone-bin + configMap: + name: keystone-bin + {{- if $mounts_keystone_credential_rotate.volumes }}{{ toYaml $mounts_keystone_credential_rotate.volumes | indent 10 }}{{ end }} +{{- end }} diff --git a/keystone/templates/deployment-api.yaml b/keystone/templates/deployment-api.yaml index 74855622e4..6a6f030933 100644 --- a/keystone/templates/deployment-api.yaml +++ b/keystone/templates/deployment-api.yaml @@ -98,6 +98,8 @@ spec: - name: keystone-fernet-keys mountPath: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository }} {{- end }} + - name: keystone-credential-keys + mountPath: {{ .Values.conf.keystone.credential.keystone.key_repository }} {{- if $mounts_keystone_api.volumeMounts }}{{ toYaml $mounts_keystone_api.volumeMounts | indent 10 }}{{ end }} volumes: - name: etckeystone @@ -117,4 +119,7 @@ spec: secret: secretName: keystone-fernet-keys {{- end }} + - name: keystone-credential-keys + secret: + secretName: keystone-credential-keys {{- if $mounts_keystone_api.volumes }}{{ toYaml $mounts_keystone_api.volumes | indent 6 }}{{ end }} diff --git a/keystone/templates/job-credential-setup.yaml b/keystone/templates/job-credential-setup.yaml new file mode 100644 index 0000000000..30d4319f77 --- /dev/null +++ b/keystone/templates/job-credential-setup.yaml @@ -0,0 +1,70 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.credential_setup }} +{{- $mounts_keystone_credential_setup := .Values.pod.mounts.keystone_credential_setup.keystone_credential_setup }} +{{- $mounts_keystone_credential_setup_init := .Values.pod.mounts.keystone_credential_setup.init_container }} +apiVersion: batch/v1 +kind: Job +metadata: + name: keystone-credential-setup +spec: + template: + spec: + initContainers: +{{ tuple $envAll $dependencies $mounts_keystone_credential_setup_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + containers: + - name: keystone-credential-setup + image: {{ .Values.images.credential_setup }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.credential_setup | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: KEYSTONE_USER + value: {{ .Values.jobs.credential_setup.user | quote }} + - name: KEYSTONE_GROUP + value: {{ .Values.jobs.credential_setup.group | quote }} + - name: KUBERNETES_NAMESPACE + value: {{ .Release.Namespace | quote }} + - name: KEYSTONE_KEYS_REPOSITORY + value: {{ .Values.conf.keystone.credential.keystone.key_repository | quote }} + command: + - python + - /tmp/fernet-manage.py + - credential_setup + volumeMounts: + - name: etckeystone + mountPath: /etc/keystone + - name: keystone-etc + mountPath: /etc/keystone/keystone.conf + subPath: keystone.conf + readOnly: true + - name: keystone-bin + mountPath: /tmp/fernet-manage.py + subPath: fernet-manage.py + readOnly: true +{{- if $mounts_keystone_credential_setup.volumeMounts }}{{ toYaml $mounts_keystone_credential_setup.volumeMounts | indent 10 }}{{ end }} + volumes: + - name: etckeystone + emptyDir: {} + - name: keystone-etc + configMap: + name: keystone-etc + - name: keystone-bin + configMap: + name: keystone-bin +{{- if $mounts_keystone_credential_setup.volumes }}{{ toYaml $mounts_keystone_credential_setup.volumes | indent 6 }}{{ end }} diff --git a/keystone/templates/secret-credential-keys.yaml b/keystone/templates/secret-credential-keys.yaml new file mode 100644 index 0000000000..a51f2a6f67 --- /dev/null +++ b/keystone/templates/secret-credential-keys.yaml @@ -0,0 +1,20 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Secret +metadata: + name: keystone-credential-keys +type: Opaque +data: diff --git a/keystone/values.yaml b/keystone/values.yaml index 62eaf13a5d..f2775884ec 100644 --- a/keystone/values.yaml +++ b/keystone/values.yaml @@ -28,6 +28,8 @@ images: db_sync: docker.io/kolla/ubuntu-source-keystone:3.0.3 fernet_setup: docker.io/kolla/ubuntu-source-keystone:3.0.3 fernet_rotate: docker.io/kolla/ubuntu-source-keystone:3.0.3 + credential_setup: docker.io/kolla/ubuntu-source-keystone:3.0.3 + credential_rotate: docker.io/kolla/ubuntu-source-keystone:3.0.3 api: docker.io/kolla/ubuntu-source-keystone:3.0.3 dep_check: docker.io/kolla/ubuntu-source-kubernetes-entrypoint:4.0.0 pull_policy: "IfNotPresent" @@ -60,6 +62,7 @@ dependencies: api: jobs: - keystone-db-sync + - keystone-credential-setup # Comment line below when not running fernet tokens. - keystone-fernet-setup services: @@ -74,6 +77,7 @@ dependencies: db_sync: jobs: - keystone-db-init + - keystone-credential-setup # Comment line below when not running fernet tokens. - keystone-fernet-setup services: @@ -83,6 +87,10 @@ dependencies: fernet_rotate: jobs: - keystone-fernet-setup + credential_setup: + credential_rotate: + jobs: + - keystone-credential-setup tests: services: - service: identity @@ -121,6 +129,12 @@ pod: keystone_fernet_rotate: init_container: null keystone_fernet_rotate: + keystone_credential_setup: + init_container: null + keystone_credential_setup: + keystone_credential_rotate: + init_container: null + keystone_credential_rotate: replicas: api: 1 lifecycle: @@ -189,6 +203,20 @@ pod: requests: memory: "1024Mi" cpu: "2000m" + credential_setup: + limits: + memory: "128Mi" + cpu: "100m" + requests: + memory: "1024Mi" + cpu: "2000m" + credential_rotate: + limits: + memory: "128Mi" + cpu: "100m" + requests: + memory: "1024Mi" + cpu: "2000m" jobs: fernet_setup: @@ -199,6 +227,15 @@ jobs: cron: "0 0 * * 0" user: keystone group: keystone + credential_setup: + user: keystone + group: keystone + credential_rotate: + # monthly + cron: "0 0 1 * *" + migrate_wait: 120 + user: keystone + group: keystone conf: rally_tests: @@ -222,6 +259,9 @@ conf: fernet_tokens: keystone: key_repository: /etc/keystone/fernet-keys/ + credential: + keystone: + key_repository: /etc/keystone/credential-keys/ database: oslo: db: