diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install b/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install index 0095bb10a..2581dcbeb 100644 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install @@ -8,3 +8,4 @@ etc/systemd/system.conf.d/kubernetes-accounting.conf usr/lib/tmpfiles.d/kubernetes.conf usr/local/sbin/sanitize_kubelet_reserved_cpus.sh usr/local/sbin/upgrade_k8s_config.sh +usr/local/sbin/sanitize_feature_gates.py diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/rules b/kubernetes/kubernetes-unversioned/debian/deb_folder/rules index 8ab58fb08..e7209a239 100755 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/rules +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/rules @@ -66,6 +66,7 @@ override_dh_install: install -v -m 0700 -d ${DEBIAN_DESTDIR}${_local_sbindir} install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/sanitize_kubelet_reserved_cpus.sh install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/upgrade_k8s_config.sh + install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/sanitize_feature_gates.py dh_install diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/sanitize_feature_gates.py b/kubernetes/kubernetes-unversioned/debian/deb_folder/sanitize_feature_gates.py new file mode 100644 index 000000000..66f73a0aa --- /dev/null +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/sanitize_feature_gates.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +'''Removes the "RemoveSelfLink=false" kube-apiserver feature gate + +Usage: + ./sanitize_feature_gates.py + +This script is just to remove the "RemoveSelfLink=false" kube-apiserver +feature gate from the last_kube_extra_config_bootstrap.yaml file when +doing an upgrade from K8s 1.23 to 1.24. +Once we no longer need to worry about upgrading from 1.23 we can remove this. + +Copyright (c) 2022 Wind River Systems, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at: + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +OR CONDITIONS OF ANY KIND, either express or implied. +''' + +import syslog +import yaml + +FILENAME = "/opt/platform/config/22.12/last_kube_extra_config_bootstrap.yaml" + +# The above file contains information that is used during backup and restore. +# We want to remove the "RemoveSelfLink=false" kube-apiserver feature gate +# if it's present so that if we do a backup and restore after the upgrade +# to K8s 1.24 we won't try to use this feature gate any more. Any other feature +# gates should be left alone, and if there aren't any other feature gates +# we want to delete the entire "feature-gates" entry. + +# Logs will go to /var/log/daemon.log +syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_DAEMON) + +try: + with open(FILENAME, "r") as stream: + info = yaml.safe_load(stream) +except Exception as exc: + syslog.syslog(syslog.LOG_ERR, "Problem reading from {}".format(FILENAME)) + syslog.syslog(syslog.LOG_ERR, str(exc)) + exit(1) + +try: + feature_gates = info['apiserver_extra_args']['feature-gates'] +except KeyError: + # No apiserver feature gates, nothing to do + syslog.syslog('No kube-apiserver feature gates, nothing to do.') + exit(0) + +if "RemoveSelfLink=false" not in feature_gates: + # Nothing to do + syslog.syslog('No changes needed in kube-apiserver feature gates.') + exit(0) + +# Remove "RemoveSelfLink=false" from the feature gates +# we need to handle the case where it could be at the beginning of the string +# with other entries after it, or at the end of the string with other entries +# before it, in the middle of the string, or by itself. +feature_gates = feature_gates.replace('RemoveSelfLink=false,', '') +feature_gates = feature_gates.replace(',RemoveSelfLink=false', '') +feature_gates = feature_gates.replace('RemoveSelfLink=false', '') + +if not feature_gates: + # No feature gates left, so delete the entry + syslog.syslog('Deleting kube-apiserver feature gates.') + info['apiserver_extra_args'].pop('feature-gates', None) +else: + # Update the feature gates with the new value + syslog.syslog('Modifying kube-apiserver feature gates.') + info['apiserver_extra_args']['feature-gates'] = feature_gates + +# Write out the new file. +try: + with open(FILENAME, 'w') as outfile: + yaml.safe_dump(info, outfile, default_flow_style=False, sort_keys=False) +except Exception as exc: + syslog.syslog(syslog.LOG_ERR, "Problem writing to {}".format(FILENAME)) + syslog.syslog(syslog.LOG_ERR, str(exc)) + exit(1) + +syslog.syslog('Successfully wrote file with RemoveSelfLink=false removed.') diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh b/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh index 61734af3f..438b98a5c 100755 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh @@ -67,6 +67,75 @@ function get_kubeadm_configmap { fi } +# Remove the "RemoveSelfLink=false" kube-apiserver feature gate from the service parameters. +# This is needed to ensure that a backup taken after the upgrade to K8s 1.24 will be +# properly restored without this feature gate. (K8s 1.24 no longer supports this feature gate.) +function update_feature_gates_service_param_v1_24 { + + # Enable authentication. + source /etc/platform/openrc + + # Check if any kube-apiserver feature gates are specified in the service parameters. + TMP=`system service-parameter-list|grep kube_apiserver|grep feature-gates` + if [ $? -ne 0 ]; then + # no feature-gates specified, nothing to do + LOG "No kube-apiserver feature-gates service param specified, nothing to do." + return + fi + + # Get the actual feature gates. + # The xargs call here strips any whitespace. + FEATURE_GATES=`echo ${TMP}|cut -d'|' -f6|xargs` + if [ -z "${FEATURE_GATES}" ]; then + # No feature-gates, nothing to do. Really shouldn't hit this, just being paranoid. + LOG "No kube-apiserver feature-gates specified, nothing to do." + return + fi + + # Check if the specific feature gate we care about is specified. + echo ${FEATURE_GATES}|grep -q "RemoveSelfLink=false" + if [ $? -ne 0 ]; then + # RemoveSelfLink=false is not specified, nothing to do + LOG "RemoveSelfLink=false kube-apiserver feature gate not specified, nothing to do." + return + fi + + # Remove "RemoveSelfLink=false" from the feature gates. + # We need to handle the case where it could be at the beginning of the string + # with other entries after it, or at the end of the string with other entries + # before it, in the middle of the string, or by itself. + NEW_FEATURE_GATES=${FEATURE_GATES//'RemoveSelfLink=false,'/} + NEW_FEATURE_GATES=${NEW_FEATURE_GATES//',RemoveSelfLink=false'/} + NEW_FEATURE_GATES=${NEW_FEATURE_GATES//'RemoveSelfLink=false'/} + + if [ -z "${NEW_FEATURE_GATES}" ]; then + # no other feature gates, so delete the service parameter rather than modify it + UUID=`echo ${TMP}|cut -d'|' -f2` + system service-parameter-delete ${UUID} + if [ $? -eq 0 ]; then + LOG "Successfully deleted RemoveSelfLink=false feature gate service parameter ${UUID}" + else + LOG "Failed to delete RemoveSelfLink=false feature gate service parameter ${UUID}" + fi + else + # need to modify the service parameter with the new feature gates + system service-parameter-modify kubernetes kube_apiserver feature-gates=${NEW_FEATURE_GATES} + if [ $? -eq 0 ]; then + LOG "Successfully modified kube-apiserver feature gate service parameter." + else + LOG "Failed to modify kube-apiserver feature gate service parameter." + fi + fi +} + +# This needs to edit a yaml file, so we call out to a python helper. +function update_kube_extra_config_bootstrap_v1_24 { + /usr/local/sbin/sanitize_feature_gates.py + if [ $? -ne 0 ]; then + LOG "Problem running sanitize_feature_gates.py, may cause problems with backup/restore." + fi +} + # Update feature gates for version 1.24 function update_feature_gates_v1_24 { @@ -156,6 +225,8 @@ until [ ${counter} -gt ${RETRIES} ]; do if [[ "${K8S_VERSION}" == "v1.21.8" ]]; then update_feature_gates_v1_22 elif [[ "${K8S_VERSION}" == "v1.23.1" ]]; then + update_feature_gates_service_param_v1_24 + update_kube_extra_config_bootstrap_v1_24 update_feature_gates_v1_24 else LOG "No update required for kubeadm configmap"