From 03543a3d53c10710d4d7b6a8a69a208380da4f89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Dulko?= Date: Wed, 10 May 2017 09:44:29 +0200 Subject: [PATCH] Add support for Keystone's fernet tokens Keystone supports (and that's a default setting since Ocata) using non-persistent fernet tokens instead of UUID tokens written into the DB. This setting is in some cases better in terms of performance and manageability (no more tokens DB table cleanups). OpenStack-Helm should be able to support it. General issue with fernet tokens is that keys used to encrypt them need to be persistent and shared accross the cluster. Moreover "rotate" operation generates a new key, so key repository will change over time. This commit implements fernet tokens support by: * A 'keystone-fernet-keys' secret is created to serve as keys repository. * New fernet-setup Job will populate secret with initial keys. * New fernet-rotate CronJob will be run periodically (weekly by default) and perform key rotation operation and update the secret. * Secret is attached to keystone-api pods in /etc/keystone/fernet-tokens directory. Turns out k8s is updating secrets attached to pods automatically, so because of Keystone's fernet tokens implementation, we don't need to worry about synchronization of the key repository. Everything should be fine unless fernet-rotate job will run before all of the pods will notice the change in the secret. As in real-world scenario you would rotate your keys no more often than once an hour, this should be totally fine. Implements: blueprint keystone-fernet-tokens Change-Id: Ifc84b8c97e1a85d30eb46260582d9c58220fbf0a --- doc/source/install/multinode.rst | 13 ++ keystone/templates/bin/_fernet-manage.py.tpl | 162 ++++++++++++++++++ keystone/templates/configmap-bin.yaml | 2 + .../templates/cron-job-fernet-rotate.yaml | 76 ++++++++ keystone/templates/deployment-api.yaml | 9 + keystone/templates/job-db-sync.yaml | 9 + keystone/templates/job-fernet-setup.yaml | 72 ++++++++ keystone/templates/secret-fernet-keys.yaml | 22 +++ keystone/values.yaml | 47 ++++- tools/kubeadm-aio/assets/etc/kubeadm.conf | 4 + tools/kubeadm-aio/assets/usr/bin/kubeadm-aio | 4 +- 11 files changed, 416 insertions(+), 4 deletions(-) create mode 100644 keystone/templates/bin/_fernet-manage.py.tpl create mode 100644 keystone/templates/cron-job-fernet-rotate.yaml create mode 100644 keystone/templates/job-fernet-setup.yaml create mode 100644 keystone/templates/secret-fernet-keys.yaml create mode 100644 tools/kubeadm-aio/assets/etc/kubeadm.conf diff --git a/doc/source/install/multinode.rst b/doc/source/install/multinode.rst index 7a60b7ce95..088b490a9d 100644 --- a/doc/source/install/multinode.rst +++ b/doc/source/install/multinode.rst @@ -174,6 +174,19 @@ component so we relax the access control rules: kubectl update -f https://raw.githubusercontent.com/openstack/openstack-helm/master/tools/kubeadm-aio/assets/opt/rbac/dev.yaml +Enabling Cron Jobs +------------------ + +OpenStack-Helm's default Keystone token provider is `fernet +`_. +To provide sufficient security, keys used to generate fernet tokens need to be +rotated regularly. Keystone chart provides Cron Job for that task, but it is +only deployed when Cron Jobs API is enabled on Kubernetes cluster. To enable +Cron Jobs add ``--runtime-config=batch/v2alpha1=true`` to your kube-apiserver +startup arguments (e.g. in your +``/etc/kubernetes/manifests/kube-apiserver.yaml`` manifest). By default fernet +keys will be rotated weekly. + Preparing Persistent Storage ---------------------------- diff --git a/keystone/templates/bin/_fernet-manage.py.tpl b/keystone/templates/bin/_fernet-manage.py.tpl new file mode 100644 index 0000000000..4b484fae1b --- /dev/null +++ b/keystone/templates/bin/_fernet-manage.py.tpl @@ -0,0 +1,162 @@ +#!/usr/bin/env python + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import base64 +import errno +import grp +import logging +import os +import pwd +import re +import six +import subprocess +import sys + +import requests + +FERNET_DIR = os.environ['KEYSTONE_KEYS_REPOSITORY'] +KEYSTONE_USER = os.environ['KEYSTONE_USER'] +KEYSTONE_GROUP = os.environ['KEYSTONE_GROUP'] +SECRET_NAME = 'keystone-fernet-keys' +NAMESPACE = os.environ['KUBERNETES_NAMESPACE'] + +# k8s connection data +KUBE_HOST = None +KUBE_CERT = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' +KUBE_TOKEN = None + +LOG_DATEFMT = "%Y-%m-%d %H:%M:%S" +LOG_FORMAT = "%(asctime)s.%(msecs)03d - %(levelname)s - %(message)s" +logging.basicConfig(format=LOG_FORMAT, datefmt=LOG_DATEFMT) +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.INFO) + + +def read_kube_config(): + global KUBE_HOST, KUBE_TOKEN + KUBE_HOST = "https://%s:%s" % ('kubernetes.default', + os.environ['KUBERNETES_SERVICE_PORT']) + with open('/var/run/secrets/kubernetes.io/serviceaccount/token', 'r') as f: + KUBE_TOKEN = f.read() + + +def get_secret_definition(name): + url = '%s/api/v1/namespaces/%s/secrets/%s' % (KUBE_HOST, NAMESPACE, name) + resp = requests.get(url, + headers={'Authorization': 'Bearer %s' % KUBE_TOKEN}, + verify=KUBE_CERT) + if resp.status_code != 200: + LOG.error('Cannot get secret %s.', name) + LOG.error(resp.text) + return None + return resp.json() + + +def update_secret(name, secret): + url = '%s/api/v1/namespaces/%s/secrets/%s' % (KUBE_HOST, NAMESPACE, name) + resp = requests.put(url, + json=secret, + headers={'Authorization': 'Bearer %s' % KUBE_TOKEN}, + verify=KUBE_CERT) + if resp.status_code != 200: + LOG.error('Cannot update secret %s.', name) + LOG.error(resp.text) + return False + return True + + +def read_from_files(): + keys = filter( + lambda name: os.path.isfile(FERNET_DIR + name) and re.match("^\d+$", + name), + os.listdir(FERNET_DIR) + ) + data = {} + for key in keys: + with open(FERNET_DIR + key, 'r') as f: + data[key] = f.read() + if len(keys): + LOG.debug("Keys read from files: %s", keys) + else: + LOG.warn("No keys were read from files.") + return data + + +def get_keys_data(): + keys = read_from_files() + return dict([(key, base64.b64encode(value.encode()).decode()) + for (key, value) in six.iteritems(keys)]) + + +def write_to_files(data): + if not os.path.exists(os.path.dirname(FERNET_DIR)): + try: + os.makedirs(os.path.dirname(FERNET_DIR)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + uid = pwd.getpwnam(KEYSTONE_USER).pw_uid + gid = grp.getgrnam(KEYSTONE_GROUP).gr_gid + os.chown(FERNET_DIR, uid, gid) + + for (key, value) in six.iteritems(data): + with open(FERNET_DIR + key, 'w') as f: + decoded_value = base64.b64decode(value).decode() + f.write(decoded_value) + LOG.debug("Key %s: %s", key, decoded_value) + LOG.info("%s keys were written", len(data)) + + +def execute_command(cmd): + LOG.info("Executing 'keystone-manage %s --keystone-user=%s " + "--keystone-group=%s' command.", + cmd, KEYSTONE_USER, KEYSTONE_GROUP) + subprocess.call(['keystone-manage', cmd, + '--keystone-user=%s' % KEYSTONE_USER, + '--keystone-group=%s' % KEYSTONE_GROUP]) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('command', choices=['fernet_setup', 'fernet_rotate']) + args = parser.parse_args() + + read_kube_config() + secret = get_secret_definition(SECRET_NAME) + if not secret: + LOG.error("Secret '%s' does not exist.", SECRET_NAME) + sys.exit(1) + + if args.command == 'fernet_rotate': + LOG.info("Copying existing fernet keys from secret '%s' to %s.", + SECRET_NAME, FERNET_DIR) + write_to_files(secret['data']) + + execute_command(args.command) + + LOG.info("Updating data for '%s' secret.", SECRET_NAME) + updated_keys = get_keys_data() + secret['data'] = updated_keys + if not update_secret(SECRET_NAME, secret): + sys.exit(1) + LOG.info("%s fernet keys have been placed to secret '%s'", + len(updated_keys), SECRET_NAME) + LOG.debug("Placed keys: %s", updated_keys) + LOG.info("Fernet keys %s has been completed", + "rotation" if args.command == 'fernet_rotate' else "generation") + +if __name__ == "__main__": + main() diff --git a/keystone/templates/configmap-bin.yaml b/keystone/templates/configmap-bin.yaml index f4a6324b54..2a7746cecd 100644 --- a/keystone/templates/configmap-bin.yaml +++ b/keystone/templates/configmap-bin.yaml @@ -31,3 +31,5 @@ data: {{ tuple "bin/_db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} keystone-api.sh: | {{ tuple "bin/_keystone-api.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + fernet-manage.py: | +{{ tuple "bin/_fernet-manage.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/keystone/templates/cron-job-fernet-rotate.yaml b/keystone/templates/cron-job-fernet-rotate.yaml new file mode 100644 index 0000000000..31129ab4ea --- /dev/null +++ b/keystone/templates/cron-job-fernet-rotate.yaml @@ -0,0 +1,76 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if and (eq .Values.conf.keystone.token.keystone.provider "fernet") (.Capabilities.APIVersions.Has "batch/v2alpha1") }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.fernet_rotate }} +{{- $mounts_keystone_fernet_rotate := .Values.pod.mounts.keystone_fernet_rotate.keystone_fernet_rotate }} +{{- $mounts_keystone_fernet_rotate_init := .Values.pod.mounts.keystone_fernet_rotate.init_container }} +apiVersion: batch/v2alpha1 +kind: CronJob +metadata: + name: keystone-fernet-rotate +spec: + schedule: {{ .Values.jobs.fernet_rotate.cron | quote }} + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + initContainers: +{{ tuple $envAll $dependencies $mounts_keystone_fernet_rotate_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + containers: + - name: keystone-fernet-rotate + image: {{ .Values.images.fernet_rotate }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.fernet_rotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} + env: + - name: KEYSTONE_USER + value: {{ .Values.jobs.fernet_rotate.user | quote }} + - name: KEYSTONE_GROUP + value: {{ .Values.jobs.fernet_rotate.group | quote }} + - name: KUBERNETES_NAMESPACE + value: {{ .Release.Namespace | quote }} + - name: KEYSTONE_KEYS_REPOSITORY + value: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository | quote }} + command: + - python + - /tmp/fernet-manage.py + - fernet_rotate + volumeMounts: + - name: etckeystone + mountPath: /etc/keystone + - name: keystone-etc + mountPath: /etc/keystone/keystone.conf + subPath: keystone.conf + readOnly: true + - name: keystone-bin + mountPath: /tmp/fernet-manage.py + subPath: fernet-manage.py + readOnly: true + {{- if $mounts_keystone_fernet_rotate.volumeMounts }}{{ toYaml $mounts_keystone_fernet_rotate.volumeMounts | indent 14 }}{{ end }} + volumes: + - name: etckeystone + emptyDir: {} + - name: keystone-etc + configMap: + name: keystone-etc + - name: keystone-bin + configMap: + name: keystone-bin + {{- if $mounts_keystone_fernet_rotate.volumes }}{{ toYaml $mounts_keystone_fernet_rotate.volumes | indent 10 }}{{ end }} +{{- end }} diff --git a/keystone/templates/deployment-api.yaml b/keystone/templates/deployment-api.yaml index bde48ca58a..74855622e4 100644 --- a/keystone/templates/deployment-api.yaml +++ b/keystone/templates/deployment-api.yaml @@ -94,6 +94,10 @@ spec: mountPath: /tmp/keystone-api.sh subPath: keystone-api.sh readOnly: true +{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }} + - name: keystone-fernet-keys + mountPath: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository }} +{{- end }} {{- if $mounts_keystone_api.volumeMounts }}{{ toYaml $mounts_keystone_api.volumeMounts | indent 10 }}{{ end }} volumes: - name: etckeystone @@ -108,4 +112,9 @@ spec: configMap: name: keystone-bin defaultMode: 0555 +{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }} + - name: keystone-fernet-keys + secret: + secretName: keystone-fernet-keys +{{- end }} {{- if $mounts_keystone_api.volumes }}{{ toYaml $mounts_keystone_api.volumes | indent 6 }}{{ end }} diff --git a/keystone/templates/job-db-sync.yaml b/keystone/templates/job-db-sync.yaml index d884f20605..b56012846c 100644 --- a/keystone/templates/job-db-sync.yaml +++ b/keystone/templates/job-db-sync.yaml @@ -59,6 +59,10 @@ spec: mountPath: /tmp/db-sync.sh subPath: db-sync.sh readOnly: true +{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }} + - name: keystone-fernet-keys + mountPath: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository }} +{{- end }} {{- if $mounts_keystone_db_sync.volumeMounts }}{{ toYaml $mounts_keystone_db_sync.volumeMounts | indent 10 }}{{ end }} volumes: - name: etckeystone @@ -71,4 +75,9 @@ spec: configMap: name: keystone-bin defaultMode: 0555 +{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }} + - name: keystone-fernet-keys + secret: + secretName: keystone-fernet-keys +{{- end }} {{- if $mounts_keystone_db_sync.volumes }}{{ toYaml $mounts_keystone_db_sync.volumes | indent 6 }}{{ end }} diff --git a/keystone/templates/job-fernet-setup.yaml b/keystone/templates/job-fernet-setup.yaml new file mode 100644 index 0000000000..5792bdc3a3 --- /dev/null +++ b/keystone/templates/job-fernet-setup.yaml @@ -0,0 +1,72 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.fernet_setup }} +{{- $mounts_keystone_fernet_setup := .Values.pod.mounts.keystone_fernet_setup.keystone_fernet_setup }} +{{- $mounts_keystone_fernet_setup_init := .Values.pod.mounts.keystone_fernet_setup.init_container }} +apiVersion: batch/v1 +kind: Job +metadata: + name: keystone-fernet-setup +spec: + template: + spec: + initContainers: +{{ tuple $envAll $dependencies $mounts_keystone_fernet_setup_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + containers: + - name: keystone-fernet-setup + image: {{ .Values.images.fernet_setup }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.fernet_setup | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: KEYSTONE_USER + value: {{ .Values.jobs.fernet_setup.user | quote }} + - name: KEYSTONE_GROUP + value: {{ .Values.jobs.fernet_setup.group | quote }} + - name: KUBERNETES_NAMESPACE + value: {{ .Release.Namespace | quote }} + - name: KEYSTONE_KEYS_REPOSITORY + value: {{ .Values.conf.keystone.fernet_tokens.keystone.key_repository | quote }} + command: + - python + - /tmp/fernet-manage.py + - fernet_setup + volumeMounts: + - name: etckeystone + mountPath: /etc/keystone + - name: keystone-etc + mountPath: /etc/keystone/keystone.conf + subPath: keystone.conf + readOnly: true + - name: keystone-bin + mountPath: /tmp/fernet-manage.py + subPath: fernet-manage.py + readOnly: true +{{- if $mounts_keystone_fernet_setup.volumeMounts }}{{ toYaml $mounts_keystone_fernet_setup.volumeMounts | indent 10 }}{{ end }} + volumes: + - name: etckeystone + emptyDir: {} + - name: keystone-etc + configMap: + name: keystone-etc + - name: keystone-bin + configMap: + name: keystone-bin +{{- if $mounts_keystone_fernet_setup.volumes }}{{ toYaml $mounts_keystone_fernet_setup.volumes | indent 6 }}{{ end }} +{{- end }} diff --git a/keystone/templates/secret-fernet-keys.yaml b/keystone/templates/secret-fernet-keys.yaml new file mode 100644 index 0000000000..4f4367b531 --- /dev/null +++ b/keystone/templates/secret-fernet-keys.yaml @@ -0,0 +1,22 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if eq .Values.conf.keystone.token.keystone.provider "fernet" }} +apiVersion: v1 +kind: Secret +metadata: + name: keystone-fernet-keys +type: Opaque +data: +{{- end }} diff --git a/keystone/values.yaml b/keystone/values.yaml index 285d514d6e..62eaf13a5d 100644 --- a/keystone/values.yaml +++ b/keystone/values.yaml @@ -26,6 +26,8 @@ images: test: docker.io/kolla/ubuntu-binary-rally:4.0.0 db_init: docker.io/kolla/ubuntu-source-keystone:3.0.3 db_sync: docker.io/kolla/ubuntu-source-keystone:3.0.3 + fernet_setup: docker.io/kolla/ubuntu-source-keystone:3.0.3 + fernet_rotate: docker.io/kolla/ubuntu-source-keystone:3.0.3 api: docker.io/kolla/ubuntu-source-keystone:3.0.3 dep_check: docker.io/kolla/ubuntu-source-kubernetes-entrypoint:4.0.0 pull_policy: "IfNotPresent" @@ -58,6 +60,8 @@ dependencies: api: jobs: - keystone-db-sync + # Comment line below when not running fernet tokens. + - keystone-fernet-setup services: - service: oslo_cache endpoint: internal @@ -70,9 +74,15 @@ dependencies: db_sync: jobs: - keystone-db-init + # Comment line below when not running fernet tokens. + - keystone-fernet-setup services: - service: oslo_db endpoint: internal + fernet_setup: + fernet_rotate: + jobs: + - keystone-fernet-setup tests: services: - service: identity @@ -105,6 +115,12 @@ pod: keystone_bootstrap: init_container: null keystone_bootstrap: + keystone_fernet_setup: + init_container: null + keystone_fernet_setup: + keystone_fernet_rotate: + init_container: null + keystone_fernet_rotate: replicas: api: 1 lifecycle: @@ -159,6 +175,30 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + fernet_setup: + limits: + memory: "128Mi" + cpu: "100m" + requests: + memory: "10024Mi" + cpu: "2000m" + fernet_rotate: + limits: + memory: "128Mi" + cpu: "100m" + requests: + memory: "1024Mi" + cpu: "2000m" + +jobs: + fernet_setup: + user: keystone + group: keystone + fernet_rotate: + # weekly + cron: "0 0 * * 0" + user: keystone + group: keystone conf: rally_tests: @@ -175,10 +215,13 @@ conf: append: default: keystone: - max_token_size: 32 + max_token_size: 255 token: keystone: - provider: uuid + provider: fernet + fernet_tokens: + keystone: + key_repository: /etc/keystone/fernet-keys/ database: oslo: db: diff --git a/tools/kubeadm-aio/assets/etc/kubeadm.conf b/tools/kubeadm-aio/assets/etc/kubeadm.conf new file mode 100644 index 0000000000..af63053570 --- /dev/null +++ b/tools/kubeadm-aio/assets/etc/kubeadm.conf @@ -0,0 +1,4 @@ +apiVersion: kubeadm.k8s.io/v1alpha1 +kind: MasterConfiguration +apiServerExtraArgs: + runtime-config: "batch/v2alpha1=true" diff --git a/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio b/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio index 082240a4e8..6655e77d18 100755 --- a/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio +++ b/tools/kubeadm-aio/assets/usr/bin/kubeadm-aio @@ -31,9 +31,9 @@ if [[ "${KUBE_ROLE}" == "master" ]]; then if [[ "$KUBE_BIND_DEV" != "autodetect" ]]; then KUBE_BIND_IP=$(ip addr list ${KUBE_BIND_DEV} |grep "inet " |cut -d' ' -f6|cut -d/ -f1) echo 'We are going to bind the K8s API to: ${KUBE_BIND_IP}' - kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} --api-advertise-addresses ${KUBE_BIND_IP} + kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} --api-advertise-addresses ${KUBE_BIND_IP} --config /etc/kubeadm.conf else - kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} + kubeadm init --skip-preflight-checks ${KUBE_VERSION_FLAG} --config /etc/kubeadm.conf fi echo 'Setting up K8s client'