Build fernet keys inside Kubernetes

This patch moves fernet keys to be managed directly as native Kubernetes
secrets with rotations.

Change-Id: Ib6b5e3b65736eb882222ff28719e39e5d8a4f8cd

devstack/lib/keystone

@@ -36,18 +36,6 @@ function init_keystone {
 	sudo docker run -v /etc/keystone:/etc/keystone vexxhost/keystone:latest keystone-manage --config-file $KEYSTONE_CONF db_sync
 	time_stop "dbsync"
 
-	# Get fernet keys
-	if [[ "$KEYSTONE_TOKEN_FORMAT" == "fernet" ]]; then
-		rm -rf "$KEYSTONE_CONF_DIR/fernet-keys/"
-		mkdir "$KEYSTONE_CONF_DIR/fernet-keys/"
-		sudo chmod -Rv 777 "$KEYSTONE_CONF_DIR/fernet-keys/"
-		sudo docker run -v /etc/keystone:/etc/keystone vexxhost/keystone:latest keystone-manage --config-file $KEYSTONE_CONF fernet_setup --keystone-user 65534 --keystone-group 65534
-	fi
-
-	# Get credential keys
-	rm -rf "$KEYSTONE_CONF_DIR/credential-keys/"
-	sudo docker run -v /etc/keystone:/etc/keystone vexxhost/keystone:latest keystone-manage --config-file $KEYSTONE_CONF credential_setup --keystone-user 65534 --keystone-group 65534
-
 }
 export -f init_keystone
 
@@ -112,4 +100,4 @@ function bootstrap_keystone {
 		--bootstrap-admin-url "$KEYSTONE_AUTH_URI" \
 		--bootstrap-public-url "$KEYSTONE_SERVICE_URI"
 }
-export -f bootstrap_keystone
+export -f bootstrap_keystone

openstack_operator/filters.py

@@ -0,0 +1,25 @@
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Kopf filters
+
+This module contains a few common filters to be used throughout the operator
+in order to reduce strain on the API server.
+"""
+
+
+def managed(namespace, labels, **_):
+    """Check if a resource is managed by the operator."""
+    return namespace == 'openstack' and \
+        labels.get('app.kubernetes.io/managed-by') == 'openstack-operator'

openstack_operator/keystone.py

@@ -17,10 +17,73 @@
 This module maintains the operator for Keystone which does everything from
 deployment to taking care of rotating fernet & credentials keys."""
 
+import base64
 import kopf
 
+from cryptography import fernet
+
+from openstack_operator import filters
 from openstack_operator import utils
 
+TOKEN_EXPIRATION = 86400
+FERNET_ROTATION_INTERVAL = 3600
+
+
+def _is_keystone_deployment(name, **_):
+    return name == 'keystone'
+
+
+def create_or_rotate_fernet_repository(name):
+    """Create or rotate fernet tokens
+
+    This will happen when it sees a Keystone deployment that we manage and it
+    will initialize (or rotate) the fernet repository.
+    """
+
+    data = utils.get_secret('openstack', 'keystone-%s' % (name))
+
+    # Stage an initial key 0 if we don't have anything.
+    if data is None:
+        data = {'0': fernet.Fernet.generate_key().decode('utf-8')}
+
+    # Get highest key number
+    sorted_keys = [int(k) for k in data.keys()]
+    sorted_keys.sort()
+    next_key = str(max(sorted_keys) + 1)
+
+    # Promote key 0 to primary
+    data[next_key] = data['0']
+    sorted_keys.append(int(next_key))
+
+    # Stage a new key
+    data['0'] = fernet.Fernet.generate_key().decode('utf-8')
+
+    # Determine number of active keys
+    active_keys = int(TOKEN_EXPIRATION / FERNET_ROTATION_INTERVAL)
+
+    # Determine the keys to keep and drop others
+    keys_to_keep = [0] + sorted_keys[-active_keys:]
+    keys = {k: base64.b64encode(v.encode('utf-8')).decode('utf-8')
+            for k, v in data.items() if int(k) in keys_to_keep}
+
+    # Update secret
+    utils.create_or_update('keystone/secret-fernet.yml.j2', name=name,
+                           keys=keys, is_strategic=False, adopt=True)
+
+
+@kopf.timer('apps', 'v1', 'deployments',
+            when=kopf.all_([filters.managed, _is_keystone_deployment]),
+            interval=FERNET_ROTATION_INTERVAL)
+def create_or_rotate_fernet(**_):
+    """Create or rotate fernet keys
+
+    This will happen when it sees a Keystone deployment that we manage and it
+    will initialize (or rotate) the fernet repository.
+    """
+
+    create_or_rotate_fernet_repository('fernet')
+    create_or_rotate_fernet_repository('credential')
+
 
 @kopf.on.resume('identity.openstack.org', 'v1alpha1', 'keystones')
 @kopf.on.create('identity.openstack.org', 'v1alpha1', 'keystones')

openstack_operator/templates/keystone/deployment.yml.j2

@@ -65,11 +65,21 @@ spec:
         volumeMounts:
         - mountPath: /etc/keystone
           name: config
+        - name: fernet-keys
+          mountPath: /etc/keystone/fernet-keys
+        - name: credential-keys
+          mountPath: /etc/keystone/credential-keys
       volumes:
       - name: config
         hostPath:
           path: {{ spec['configDir'] }}
           type: Directory
+      - name: fernet-keys
+        secret:
+          secretName: keystone-fernet
+      - name: credential-keys
+        secret:
+          secretName: keystone-credential
 {% if 'nodeSelector' in spec %}
       nodeSelector:
         {{ spec.nodeSelector | to_yaml | indent(8) }}

openstack_operator/templates/keystone/secret-fernet.yml.j2

@@ -0,0 +1,25 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: keystone-{{ name }}
+  namespace: openstack
+data:
+{% if keys | length > 2 %}
+  $patch: replace
+{% endif %}
+  {{ keys | to_yaml | indent(2) }}

openstack_operator/utils.py

@@ -20,6 +20,7 @@ to be able to use them across all different operators.
 import base64
 import copy
 import operator
+import json
 import os
 import secrets
 import string
@@ -28,6 +29,7 @@ import jinja2
 import kopf
 from pbr import version
 import pykube
+from pykube.utils import obj_merge
 import yaml
 import openstack
 
@@ -85,7 +87,7 @@ ENV.filters['to_yaml'] = to_yaml
 ENV.globals['labels'] = labels
 
 
-def create_or_update(template, **kwargs):
+def create_or_update(template, is_strategic=True, **kwargs):
     """Create or update a Kubernetes resource.
 
     This function is called with a template and the args to pass to that
@@ -101,7 +103,23 @@ def create_or_update(template, **kwargs):
     try:
         resource.reload()
         resource.obj = obj
-        resource.update()
+
+        # NOTE(mnaser): Workaround until the following lands
+        #               https://github.com/hjacobs/pykube/pull/68
+        # pylint: disable=W0212
+        patch = obj_merge(resource.obj, resource._original_obj, is_strategic)
+        resp = resource.api.patch(
+            **resource.api_kwargs(
+                headers={
+                    "Content-Type": "application/strategic-merge-patch+json"
+                },
+                data=json.dumps(patch),
+            )
+        )
+        resource.api.raise_for_status(resp)
+        resource.set_obj(resp.json())
+
+        resource.update(is_strategic)
     except pykube.exceptions.HTTPError as exc:
         if exc.code != 404:
             raise

requirements.txt

@@ -1,3 +1,3 @@
-kopf
+kopf==0.27rc6
 Jinja2
 openstacksdk

tox.ini

@@ -33,6 +33,7 @@ commands =
   kopf run {posargs}
 
 [testenv:docs]
+skip_install = true
 deps =
   -r{toxinidir}/doc/requirements.txt
 commands =