diff --git a/doc/source/install/multinode.rst b/doc/source/install/multinode.rst index 7a60b7ce95..088b490a9d 100644 --- a/doc/source/install/multinode.rst +++ b/doc/source/install/multinode.rst @@ -174,6 +174,19 @@ component so we relax the access control rules: kubectl update -f https://raw.githubusercontent.com/openstack/openstack-helm/master/tools/kubeadm-aio/assets/opt/rbac/dev.yaml +Enabling Cron Jobs +------------------ + +OpenStack-Helm's default Keystone token provider is `fernet +`_. +To provide sufficient security, keys used to generate fernet tokens need to be +rotated regularly. Keystone chart provides Cron Job for that task, but it is +only deployed when Cron Jobs API is enabled on Kubernetes cluster. To enable +Cron Jobs add ``--runtime-config=batch/v2alpha1=true`` to your kube-apiserver +startup arguments (e.g. in your +``/etc/kubernetes/manifests/kube-apiserver.yaml`` manifest). By default fernet +keys will be rotated weekly. + Preparing Persistent Storage ---------------------------- diff --git a/keystone/templates/bin/_fernet-manage.py.tpl b/keystone/templates/bin/_fernet-manage.py.tpl new file mode 100644 index 0000000000..4b484fae1b --- /dev/null +++ b/keystone/templates/bin/_fernet-manage.py.tpl @@ -0,0 +1,162 @@ +#!/usr/bin/env python + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import base64 +import errno +import grp +import logging +import os +import pwd +import re +import six +import subprocess +import sys + +import requests + +FERNET_DIR = os.environ['KEYSTONE_KEYS_REPOSITORY'] +KEYSTONE_USER = os.environ['KEYSTONE_USER'] +KEYSTONE_GROUP = os.environ['KEYSTONE_GROUP'] +SECRET_NAME = 'keystone-fernet-keys' +NAMESPACE = os.environ['KUBERNETES_NAMESPACE'] + +# k8s connection data +KUBE_HOST = None +KUBE_CERT = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' +KUBE_TOKEN = None + +LOG_DATEFMT = "%Y-%m-%d %H:%M:%S" +LOG_FORMAT = "%(asctime)s.%(msecs)03d - %(levelname)s - %(message)s" +logging.basicConfig(format=LOG_FORMAT, datefmt=LOG_DATEFMT) +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.INFO) + + +def read_kube_config(): + global KUBE_HOST, KUBE_TOKEN + KUBE_HOST = "https://%s:%s" % ('kubernetes.default', + os.environ['KUBERNETES_SERVICE_PORT']) + with open('/var/run/secrets/kubernetes.io/serviceaccount/token', 'r') as f: + KUBE_TOKEN = f.read() + + +def get_secret_definition(name): + url = '%s/api/v1/namespaces/%s/secrets/%s' % (KUBE_HOST, NAMESPACE, name) + resp = requests.get(url, + headers={'Authorization': 'Bearer %s' % KUBE_TOKEN}, + verify=KUBE_CERT) + if resp.status_code != 200: + LOG.error('Cannot get secret %s.', name) + LOG.error(resp.text) + return None + return resp.json() + + +def update_secret(name, secret): + url = '%s/api/v1/namespaces/%s/secrets/%s' % (KUBE_HOST, NAMESPACE, name) + resp = requests.put(url, + json=secret, + headers={'Authorization': 'Bearer %s' % KUBE_TOKEN}, + verify=KUBE_CERT) + if resp.status_code != 200: + LOG.error('Cannot update secret %s.', name) + LOG.error(resp.text) + return False + return True + + +def read_from_files(): + keys = filter( + lambda name: os.path.isfile(FERNET_DIR + name) and re.match("^\d+$", + name), + os.listdir(FERNET_DIR) + ) + data = {} + for key in keys: + with open(FERNET_DIR + key, 'r') as f: + data[key] = f.read() + if len(keys): + LOG.debug("Keys read from files: %s", keys) + else: + LOG.warn("No keys were read from files.") + return data + + +def get_keys_data(): + keys = read_from_files() + return dict([(key, base64.b64encode(value.encode()).decode()) + for (key, value) in six.iteritems(keys)]) + + +def write_to_files(data): + if not os.path.exists(os.path.dirname(FERNET_DIR)): + try: + os.makedirs(os.path.dirname(FERNET_DIR)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + uid = pwd.getpwnam(KEYSTONE_USER).pw_uid + gid = grp.getgrnam(KEYSTONE_GROUP).gr_gid + os.chown(FERNET_DIR, uid, gid) + + for (key, value) in six.iteritems(data): + with open(FERNET_DIR + key, 'w') as f: + decoded_value = base64.b64decode(value).decode() + f.write(decoded_value) + LOG.debug("Key %s: %s", key, decoded_value) + LOG.info("%s keys were written", len(data)) + + +def execute_command(cmd): + LOG.info("Executing 'keystone-manage %s --keystone-user=%s " + "--keystone-group=%s' command.", + cmd, KEYSTONE_USER, KEYSTONE_GROUP) + subprocess.call(['keystone-manage', cmd, + '--keystone-user=%s' % KEYSTONE_USER, + '--keystone-group=%s' % KEYSTONE_GROUP]) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('command', choices=['fernet_setup', 'fernet_rotate']) + args = parser.parse_args() + + read_kube_config() + secret = get_secret_definition(SECRET_NAME) + if not secret: + LOG.error("Secret '%s' does not exist.", SECRET_NAME) + sys.exit(1) + + if args.command == 'fernet_rotate': + LOG.info("Copying existing fernet keys from secret '%s' to %s.", + SECRET_NAME, FERNET_DIR) + write_to_files(secret['data']) + + execute_command(args.command) + + LOG.info("Updating data for '%s' secret.", SECRET_NAME) + updated_keys = get_keys_data() + secret['data'] = updated_keys + if not update_secret(SECRET_NAME, secret): + sys.exit(1) + LOG.info("%s fernet keys have been placed to secret '%s'", + len(updated_keys), SECRET_NAME) + LOG.debug("Placed keys: %s", updated_keys) + LOG.info("Fernet keys %s has been completed", + "rotation" if args.command == 'fernet_rotate' else "generation") + +if __name__ == "__main__": + main() diff --git a/keystone/templates/configmap-bin.yaml b/keystone/templates/configmap-bin.yaml index f4a6324b54..2a7746cecd 100644 --- a/keystone/templates/configmap-bin.yaml +++ b/keystone/templates/configmap-bin.yaml @@ -31,3 +31,5 @@ data: {{ tuple "bin/_db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} keystone-api.sh: | {{ tuple "bin/_keystone-api.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + fernet-manage.py: | +{{ tuple "bin/_fernet-manage.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/keystone/templates/cron-job-fernet-rotate.yaml b/keystone/templates/cron-job-fernet-rotate.yaml new file mode 100644 index 0000000000..31129ab4ea --- /dev/null +++ b/keystone/templates/cron-job-fernet-rotate.yaml @@ -0,0 +1,76 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if and (eq .Values.conf.keystone.token.keystone.provider "fernet") (.Capabilities.APIVersions.Has "batch/v2alpha1") }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.fernet_rotate }} +{{- $mounts_keystone_fernet_rotate := .Values.pod.mounts.keystone_fernet_rotate.keystone_fernet_rotate }} +{{- $mounts_keystone_fernet_rotate_init := .Values.pod.mounts.keystone_fernet_rotate.init_container }} +apiVersion: batch/v2alpha1 +kind: CronJob +metadata: + name: keystone-fernet-rotate +spec: + schedule: {{ .Values.jobs.fernet_rotate.cron | quote }} + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + initContainers: +{{ tuple $envAll $dependencies $mounts_keystone_fernet_rotate_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + containers: + - name: keystone-fernet-rotate + image: {{ .Values.images.fernet_rotate }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.fernet_rotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} + env: + - name: KEYSTONE_USER + value: {{ .Values.jobs.fernet_rotate.user | quote }} + - name: KEYSTONE_GROUP + value: {{ .Values.jobs.fernet_rotate.group | quote }} + - name: KUBERNETES_NAMESPACE + value: {{ .Release.Namespace | quote }} + - name: KEYSTONE_KEYS_REPOSITORY + value: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository | quote }} + command: + - python + - /tmp/fernet-manage.py + - fernet_rotate + volumeMounts: + - name: etckeystone + mountPath: /etc/keystone + - name: keystone-etc + mountPath: /etc/keystone/keystone.conf + subPath: keystone.conf + readOnly: true + - name: keystone-bin + mountPath: /tmp/fernet-manage.py + subPath: fernet-manage.py + readOnly: true + {{- if $mounts_keystone_fernet_rotate.volumeMounts }}{{ toYaml $mounts_keystone_fernet_rotate.volumeMounts | indent 14 }}{{ end }} + volumes: + - name: etckeystone + emptyDir: {} + - name: keystone-etc + configMap: + name: keystone-etc + - name: keystone-bin + configMap: + name: keystone-bin + {{- if $mounts_keystone_fernet_rotate.volumes }}{{ toYaml $mounts_keystone_fernet_rotate.volumes | indent 10 }}{{ end }} +{{- end }} diff --git a/keystone/templates/deployment-api.yaml b/keystone/templates/deployment-api.yaml index bde48ca58a..74855622e4 100644 --- a/keystone/templates/deployment-api.yaml +++ b/keystone/templates/deployment-api.yaml @@ -94,6 +94,10 @@ spec: mountPath: /tmp/keystone-api.sh subPath: keystone-api.sh readOnly: true +{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }} + - name: keystone-fernet-keys + mountPath: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository }} +{{- end }} {{- if $mounts_keystone_api.volumeMounts }}{{ toYaml $mounts_keystone_api.volumeMounts | indent 10 }}{{ end }} volumes: - name: etckeystone @@ -108,4 +112,9 @@ spec: configMap: name: keystone-bin defaultMode: 0555 +{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }} + - name: keystone-fernet-keys + secret: + secretName: keystone-fernet-keys +{{- end }} {{- if $mounts_keystone_api.volumes }}{{ toYaml $mounts_keystone_api.volumes | indent 6 }}{{ end }} diff --git a/keystone/templates/job-db-sync.yaml b/keystone/templates/job-db-sync.yaml index d884f20605..b56012846c 100644 --- a/keystone/templates/job-db-sync.yaml +++ b/keystone/templates/job-db-sync.yaml @@ -59,6 +59,10 @@ spec: mountPath: /tmp/db-sync.sh subPath: db-sync.sh readOnly: true +{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }} + - name: keystone-fernet-keys + mountPath: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository }} +{{- end }} {{- if $mounts_keystone_db_sync.volumeMounts }}{{ toYaml $mounts_keystone_db_sync.volumeMounts | indent 10 }}{{ end }} volumes: - name: etckeystone @@ -71,4 +75,9 @@ spec: configMap: name: keystone-bin defaultMode: 0555 +{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }} + - name: keystone-fernet-keys + secret: + secretName: keystone-fernet-keys +{{- end }} {{- if $mounts_keystone_db_sync.volumes }}{{ toYaml $mounts_keystone_db_sync.volumes | indent 6 }}{{ end }} diff --git a/keystone/templates/job-fernet-setup.yaml b/keystone/templates/job-fernet-setup.yaml new file mode 100644 index 0000000000..5792bdc3a3 --- /dev/null +++ b/keystone/templates/job-fernet-setup.yaml @@ -0,0 +1,72 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.fernet_setup }} +{{- $mounts_keystone_fernet_setup := .Values.pod.mounts.keystone_fernet_setup.keystone_fernet_setup }} +{{- $mounts_keystone_fernet_setup_init := .Values.pod.mounts.keystone_fernet_setup.init_container }} +apiVersion: batch/v1 +kind: Job +metadata: + name: keystone-fernet-setup +spec: + template: + spec: + initContainers: +{{ tuple $envAll $dependencies $mounts_keystone_fernet_setup_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + containers: + - name: keystone-fernet-setup + image: {{ .Values.images.fernet_setup }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.fernet_setup | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: KEYSTONE_USER + value: {{ .Values.jobs.fernet_setup.user | quote }} + - name: KEYSTONE_GROUP + value: {{ .Values.jobs.fernet_setup.group | quote }} + - name: KUBERNETES_NAMESPACE + value: {{ .Release.Namespace | quote }} + - name: KEYSTONE_KEYS_REPOSITORY + value: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository | quote }} + command: + - python + - /tmp/fernet-manage.py + - fernet_setup + volumeMounts: + - name: etckeystone + mountPath: /etc/keystone + - name: keystone-etc + mountPath: /etc/keystone/keystone.conf + subPath: keystone.conf + readOnly: true + - name: keystone-bin + mountPath: /tmp/fernet-manage.py + subPath: fernet-manage.py + readOnly: true +{{- if $mounts_keystone_fernet_setup.volumeMounts }}{{ toYaml $mounts_keystone_fernet_setup.volumeMounts | indent 10 }}{{ end }} + volumes: + - name: etckeystone + emptyDir: {} + - name: keystone-etc + configMap: + name: keystone-etc + - name: keystone-bin + configMap: + name: keystone-bin +{{- if $mounts_keystone_fernet_setup.volumes }}{{ toYaml $mounts_keystone_fernet_setup.volumes | indent 6 }}{{ end }} +{{- end }} diff --git a/keystone/templates/secret-fernet-keys.yaml b/keystone/templates/secret-fernet-keys.yaml new file mode 100644 index 0000000000..4f4367b531 --- /dev/null +++ b/keystone/templates/secret-fernet-keys.yaml @@ -0,0 +1,22 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }} +apiVersion: v1 +kind: Secret +metadata: + name: keystone-fernet-keys +type: Opaque +data: +{{- end }} diff --git a/keystone/values.yaml b/keystone/values.yaml index 285d514d6e..62eaf13a5d 100644 --- a/keystone/values.yaml +++ b/keystone/values.yaml @@ -26,6 +26,8 @@ images: test: docker.io/kolla/ubuntu-binary-rally:4.0.0 db_init: docker.io/kolla/ubuntu-source-keystone:3.0.3 db_sync: docker.io/kolla/ubuntu-source-keystone:3.0.3 + fernet_setup: docker.io/kolla/ubuntu-source-keystone:3.0.3 + fernet_rotate: docker.io/kolla/ubuntu-source-keystone:3.0.3 api: docker.io/kolla/ubuntu-source-keystone:3.0.3 dep_check: docker.io/kolla/ubuntu-source-kubernetes-entrypoint:4.0.0 pull_policy: "IfNotPresent" @@ -58,6 +60,8 @@ dependencies: api: jobs: - keystone-db-sync + # Comment line below when not running fernet tokens. + - keystone-fernet-setup services: - service: oslo_cache endpoint: internal @@ -70,9 +74,15 @@ dependencies: db_sync: jobs: - keystone-db-init + # Comment line below when not running fernet tokens. + - keystone-fernet-setup services: - service: oslo_db endpoint: internal + fernet_setup: + fernet_rotate: + jobs: + - keystone-fernet-setup tests: services: - service: identity @@ -105,6 +115,12 @@ pod: keystone_bootstrap: init_container: null keystone_bootstrap: + keystone_fernet_setup: + init_container: null + keystone_fernet_setup: + keystone_fernet_rotate: + init_container: null + keystone_fernet_rotate: replicas: api: 1 lifecycle: @@ -159,6 +175,30 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + fernet_setup: + limits: + memory: "128Mi" + cpu: "100m" + requests: + memory: "10024Mi" + cpu: "2000m" + fernet_rotate: + limits: + memory: "128Mi" + cpu: "100m" + requests: + memory: "1024Mi" + cpu: "2000m" + +jobs: + fernet_setup: + user: keystone + group: keystone + fernet_rotate: + # weekly + cron: "0 0 * * 0" + user: keystone + group: keystone conf: rally_tests: @@ -175,10 +215,13 @@ conf: append: default: keystone: - max_token_size: 32 + max_token_size: 255 token: keystone: - provider: uuid + provider: fernet + fernet_tokens: + keystone: + key_repository: /etc/keystone/fernet-keys/ database: oslo: db: diff --git a/tools/kubeadm-aio/assets/etc/kubeadm.conf b/tools/kubeadm-aio/assets/etc/kubeadm.conf new file mode 100644 index 0000000000..af63053570 --- /dev/null +++ b/tools/kubeadm-aio/assets/etc/kubeadm.conf @@ -0,0 +1,4 @@ +apiVersion: kubeadm.k8s.io/v1alpha1 +kind: MasterConfiguration +apiServerExtraArgs: + runtime-config: "batch/v2alpha1=true" diff --git a/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio b/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio index 082240a4e8..6655e77d18 100755 --- a/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio +++ b/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio @@ -31,9 +31,9 @@ if [[ "${KUBE_ROLE}" == "master" ]]; then if [[ "$KUBE_BIND_DEV" != "autodetect" ]]; then KUBE_BIND_IP=$(ip addr list ${KUBE_BIND_DEV} |grep "inet " |cut -d' ' -f6|cut -d/ -f1) echo 'We are going to bind the K8s API to: ${KUBE_BIND_IP}' - kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} --api-advertise-addresses ${KUBE_BIND_IP} + kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} --api-advertise-addresses ${KUBE_BIND_IP} --config /etc/kubeadm.conf else - kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} + kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} --config /etc/kubeadm.conf fi echo 'Setting up K8s client'