Add setup and rotate job for credential keys

Keystone is using keys to encrypt credentials saved into the database.
The mechanism is very similar to fernet tokens. This commit implements a
job setting key repository up and rotate job for those keys. All is
based on implementation of fernet tokens.

Change-Id: I88faf1d02d2b317563e8603cebba542f8b133c6a
Closes-Bug: 1693807

doc/source/install/multinode.rst

@@ -187,6 +187,10 @@ startup arguments (e.g. in your
 ``/etc/kubernetes/manifests/kube-apiserver.yaml`` manifest). By default fernet
 keys will be rotated weekly.
 
+Please note that similar solution is used for keys used to encrypt credentials
+saved by Keystone. Those keys are also rotated by another Cron Job. By default
+it is run in a monthly manner.
+
 Preparing Persistent Storage
 ----------------------------
 

keystone/templates/bin/_fernet-manage.py.tpl

@@ -25,13 +25,13 @@ import re
 import six
 import subprocess
 import sys
+import time
 
 import requests
 
 FERNET_DIR = os.environ['KEYSTONE_KEYS_REPOSITORY']
 KEYSTONE_USER = os.environ['KEYSTONE_USER']
 KEYSTONE_GROUP = os.environ['KEYSTONE_GROUP']
-SECRET_NAME = 'keystone-fernet-keys'
 NAMESPACE = os.environ['KUBERNETES_NAMESPACE']
 
 # k8s connection data
@@ -131,18 +131,26 @@ def execute_command(cmd):
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument('command', choices=['fernet_setup', 'fernet_rotate'])
+    parser.add_argument('command', choices=['fernet_setup', 'fernet_rotate',
+                                            'credential_setup',
+                                            'credential_rotate'])
     args = parser.parse_args()
 
+    is_credential = args.command.startswith('credential')
+
+    SECRET_NAME = ('keystone-credential-keys' if is_credential else
+                   'keystone-fernet-keys')
+
     read_kube_config()
     secret = get_secret_definition(SECRET_NAME)
     if not secret:
         LOG.error("Secret '%s' does not exist.", SECRET_NAME)
         sys.exit(1)
 
-    if args.command == 'fernet_rotate':
-        LOG.info("Copying existing fernet keys from secret '%s' to %s.",
-                 SECRET_NAME, FERNET_DIR)
+    if args.command in ('fernet_rotate', 'credential_rotate'):
+        LOG.info("Copying existing %s keys from secret '%s' to %s.",
+                 'credential' if is_credential else 'fernet', SECRET_NAME,
+                 FERNET_DIR)
         write_to_files(secret['data'])
 
     execute_command(args.command)
@@ -155,8 +163,22 @@ def main():
     LOG.info("%s fernet keys have been placed to secret '%s'",
              len(updated_keys), SECRET_NAME)
     LOG.debug("Placed keys: %s", updated_keys)
-    LOG.info("Fernet keys %s has been completed",
-             "rotation" if args.command == 'fernet_rotate' else "generation")
+    LOG.info("%s keys %s has been completed",
+             "Credential" if is_credential else 'Fernet',
+             "rotation" if args.command.endswith('_rotate') else "generation")
+
+    if args.command == 'credential_rotate':
+        # `credential_rotate` needs doing `credential_migrate` as well once all
+        # of the nodes have the new keys. So we'll sleep configurable amount of
+        # time to make sure k8s reloads the secrets in all pods and then
+        # execute `credential_migrate`.
+
+        migrate_wait = os.environ['KEYSTONE_CREDENTIAL_MIGRATE_WAIT']
+        LOG.info("Waiting %d seconds to execute `credential_migrate`.",
+                 migrate_wait)
+        time.sleep(migrate_wait)
+
+        execute_command('credential_migrate')
 
 if __name__ == "__main__":
     main()

keystone/templates/cron-job-credential-rotate.yaml

@@ -0,0 +1,78 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if .Capabilities.APIVersions.Has "batch/v2alpha1"}}
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.credential_rotate }}
+{{- $mounts_keystone_credential_rotate := .Values.pod.mounts.keystone_credential_rotate.keystone_credential_rotate }}
+{{- $mounts_keystone_credential_rotate_init := .Values.pod.mounts.keystone_credential_rotate.init_container }}
+apiVersion: batch/v2alpha1
+kind: CronJob
+metadata:
+  name: keystone-credential-rotate
+spec:
+  schedule: {{ .Values.jobs.credential_rotate.cron | quote }}
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          initContainers:
+{{ tuple $envAll $dependencies $mounts_keystone_credential_rotate_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }}
+          restartPolicy: OnFailure
+          nodeSelector:
+            {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+          containers:
+            - name: keystone-credential-rotate
+              image: {{ .Values.images.credential_rotate }}
+              imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.credential_rotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
+              env:
+              - name: KEYSTONE_USER
+                value: {{ .Values.jobs.credential_rotate.user | quote }}
+              - name: KEYSTONE_GROUP
+                value: {{ .Values.jobs.credential_rotate.group | quote }}
+              - name: KUBERNETES_NAMESPACE
+                value: {{ .Release.Namespace | quote }}
+              - name: KEYSTONE_KEYS_REPOSITORY
+                value: {{ .Values.conf.keystone.credential.keystone.key_repository | quote }}
+              - name: KEYSTONE_CREDENTIAL_MIGRATE_WAIT
+                value: {{ .Values.jobs.credential_rotate.migrate_wait | quote }}
+              command:
+                - python
+                - /tmp/fernet-manage.py
+                - credential_rotate
+              volumeMounts:
+              - name: etckeystone
+                mountPath: /etc/keystone
+              - name: keystone-etc
+                mountPath: /etc/keystone/keystone.conf
+                subPath: keystone.conf
+                readOnly: true
+              - name: keystone-bin
+                mountPath: /tmp/fernet-manage.py
+                subPath: fernet-manage.py
+                readOnly: true
+    {{- if $mounts_keystone_credential_rotate.volumeMounts }}{{ toYaml $mounts_keystone_credential_rotate.volumeMounts | indent 14 }}{{ end }}
+          volumes:
+          - name: etckeystone
+            emptyDir: {}
+          - name: keystone-etc
+            configMap:
+              name: keystone-etc
+          - name: keystone-bin
+            configMap:
+              name: keystone-bin
+    {{- if $mounts_keystone_credential_rotate.volumes }}{{ toYaml $mounts_keystone_credential_rotate.volumes | indent 10 }}{{ end }}
+{{- end }}

keystone/templates/deployment-api.yaml

@@ -98,6 +98,8 @@ spec:
           - name: keystone-fernet-keys
             mountPath: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository }}
 {{- end }}
+          - name: keystone-credential-keys
+            mountPath: {{ .Values.conf.keystone.credential.keystone.key_repository }}
 {{- if $mounts_keystone_api.volumeMounts }}{{ toYaml $mounts_keystone_api.volumeMounts | indent 10 }}{{ end }}
       volumes:
       - name: etckeystone
@@ -117,4 +119,7 @@ spec:
         secret:
           secretName: keystone-fernet-keys
 {{- end }}
+      - name: keystone-credential-keys
+        secret:
+          secretName: keystone-credential-keys
 {{- if $mounts_keystone_api.volumes }}{{ toYaml $mounts_keystone_api.volumes | indent 6 }}{{ end }}

keystone/templates/job-credential-setup.yaml

@@ -0,0 +1,70 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.credential_setup }}
+{{- $mounts_keystone_credential_setup := .Values.pod.mounts.keystone_credential_setup.keystone_credential_setup }}
+{{- $mounts_keystone_credential_setup_init := .Values.pod.mounts.keystone_credential_setup.init_container }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: keystone-credential-setup
+spec:
+  template:
+    spec:
+      initContainers:
+{{ tuple $envAll $dependencies $mounts_keystone_credential_setup_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+      containers:
+        - name: keystone-credential-setup
+          image: {{ .Values.images.credential_setup }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.credential_setup | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+          - name: KEYSTONE_USER
+            value: {{ .Values.jobs.credential_setup.user | quote }}
+          - name: KEYSTONE_GROUP
+            value: {{ .Values.jobs.credential_setup.group | quote }}
+          - name: KUBERNETES_NAMESPACE
+            value: {{ .Release.Namespace | quote }}
+          - name: KEYSTONE_KEYS_REPOSITORY
+            value: {{ .Values.conf.keystone.credential.keystone.key_repository | quote }}
+          command:
+            - python
+            - /tmp/fernet-manage.py
+            - credential_setup
+          volumeMounts:
+          - name: etckeystone
+            mountPath: /etc/keystone
+          - name: keystone-etc
+            mountPath: /etc/keystone/keystone.conf
+            subPath: keystone.conf
+            readOnly: true
+          - name: keystone-bin
+            mountPath: /tmp/fernet-manage.py
+            subPath: fernet-manage.py
+            readOnly: true
+{{- if $mounts_keystone_credential_setup.volumeMounts }}{{ toYaml $mounts_keystone_credential_setup.volumeMounts | indent 10 }}{{ end }}
+      volumes:
+      - name: etckeystone
+        emptyDir: {}
+      - name: keystone-etc
+        configMap:
+          name: keystone-etc
+      - name: keystone-bin
+        configMap:
+          name: keystone-bin
+{{- if $mounts_keystone_credential_setup.volumes }}{{ toYaml $mounts_keystone_credential_setup.volumes | indent 6 }}{{ end }}

keystone/templates/secret-credential-keys.yaml

@@ -0,0 +1,20 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: keystone-credential-keys
+type: Opaque
+data:

keystone/values.yaml

@@ -28,6 +28,8 @@ images:
   db_sync: docker.io/kolla/ubuntu-source-keystone:3.0.3
   fernet_setup: docker.io/kolla/ubuntu-source-keystone:3.0.3
   fernet_rotate: docker.io/kolla/ubuntu-source-keystone:3.0.3
+  credential_setup: docker.io/kolla/ubuntu-source-keystone:3.0.3
+  credential_rotate: docker.io/kolla/ubuntu-source-keystone:3.0.3
   api: docker.io/kolla/ubuntu-source-keystone:3.0.3
   dep_check: docker.io/kolla/ubuntu-source-kubernetes-entrypoint:4.0.0
   pull_policy: "IfNotPresent"
@@ -60,6 +62,7 @@ dependencies:
   api:
     jobs:
     - keystone-db-sync
+    - keystone-credential-setup
     # Comment line below when not running fernet tokens.
     - keystone-fernet-setup
     services:
@@ -74,6 +77,7 @@ dependencies:
   db_sync:
     jobs:
     - keystone-db-init
+    - keystone-credential-setup
     # Comment line below when not running fernet tokens.
     - keystone-fernet-setup
     services:
@@ -83,6 +87,10 @@ dependencies:
   fernet_rotate:
     jobs:
     - keystone-fernet-setup
+  credential_setup:
+  credential_rotate:
+    jobs:
+    - keystone-credential-setup
   tests:
     services:
     - service: identity
@@ -121,6 +129,12 @@ pod:
     keystone_fernet_rotate:
       init_container: null
       keystone_fernet_rotate:
+    keystone_credential_setup:
+      init_container: null
+      keystone_credential_setup:
+    keystone_credential_rotate:
+      init_container: null
+      keystone_credential_rotate:
   replicas:
     api: 1
   lifecycle:
@@ -189,6 +203,20 @@ pod:
         requests:
           memory: "1024Mi"
           cpu: "2000m"
+      credential_setup:
+        limits:
+          memory: "128Mi"
+          cpu: "100m"
+        requests:
+          memory: "1024Mi"
+          cpu: "2000m"
+      credential_rotate:
+        limits:
+          memory: "128Mi"
+          cpu: "100m"
+        requests:
+          memory: "1024Mi"
+          cpu: "2000m"
 
 jobs:
   fernet_setup:
@@ -199,6 +227,15 @@ jobs:
     cron: "0 0 * * 0"
     user: keystone
     group: keystone
+  credential_setup:
+    user: keystone
+    group: keystone
+  credential_rotate:
+    # monthly
+    cron: "0 0 1 * *"
+    migrate_wait: 120
+    user: keystone
+    group: keystone
 
 conf:
   rally_tests:
@@ -222,6 +259,9 @@ conf:
     fernet_tokens:
       keystone:
         key_repository: /etc/keystone/fernet-keys/
+    credential:
+      keystone:
+        key_repository: /etc/keystone/credential-keys/
     database:
       oslo:
         db: