Merge "clean up feature gates on k8s upgrade"

kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install

@@ -8,3 +8,4 @@ etc/systemd/system.conf.d/kubernetes-accounting.conf
 usr/lib/tmpfiles.d/kubernetes.conf
 usr/local/sbin/sanitize_kubelet_reserved_cpus.sh
 usr/local/sbin/upgrade_k8s_config.sh
+usr/local/sbin/sanitize_feature_gates.py

kubernetes/kubernetes-unversioned/debian/deb_folder/rules

@@ -66,6 +66,7 @@ override_dh_install:
 	install -v -m 0700 -d ${DEBIAN_DESTDIR}${_local_sbindir}
 	install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/sanitize_kubelet_reserved_cpus.sh
 	install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/upgrade_k8s_config.sh
+	install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/sanitize_feature_gates.py
 
 	dh_install
 

kubernetes/kubernetes-unversioned/debian/deb_folder/sanitize_feature_gates.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+'''Removes the "RemoveSelfLink=false" kube-apiserver feature gate
+
+Usage:
+    ./sanitize_feature_gates.py
+
+This script is just to remove the "RemoveSelfLink=false" kube-apiserver
+feature gate from the last_kube_extra_config_bootstrap.yaml file when
+doing an upgrade from K8s 1.23 to 1.24.
+Once we no longer need to worry about upgrading from 1.23 we can remove this.
+
+Copyright (c) 2022 Wind River Systems, Inc.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at:
+      http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software  distributed
+under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+OR CONDITIONS OF ANY KIND, either express or implied. 
+'''
+
+import syslog
+import yaml
+
+FILENAME = "/opt/platform/config/22.12/last_kube_extra_config_bootstrap.yaml"
+
+# The above file contains information that is used during backup and restore.
+# We want to remove the "RemoveSelfLink=false" kube-apiserver feature gate
+# if it's present so that if we do a backup and restore after the upgrade
+# to K8s 1.24 we won't try to use this feature gate any more.  Any other feature
+# gates should be left alone, and if there aren't any other feature gates
+# we want to delete the entire "feature-gates" entry.
+
+# Logs will go to /var/log/daemon.log
+syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_DAEMON)
+
+try:
+    with open(FILENAME, "r") as stream:
+        info = yaml.safe_load(stream)
+except Exception as exc:
+    syslog.syslog(syslog.LOG_ERR, "Problem reading from {}".format(FILENAME))
+    syslog.syslog(syslog.LOG_ERR, str(exc))
+    exit(1)
+
+try:
+    feature_gates = info['apiserver_extra_args']['feature-gates']
+except KeyError:
+    # No apiserver feature gates, nothing to do
+    syslog.syslog('No kube-apiserver feature gates, nothing to do.')
+    exit(0)
+
+if "RemoveSelfLink=false" not in feature_gates:
+    # Nothing to do
+    syslog.syslog('No changes needed in kube-apiserver feature gates.')
+    exit(0)
+
+# Remove "RemoveSelfLink=false" from the feature gates
+# we need to handle the case where it could be at the beginning of the string
+# with other entries after it, or at the end of the string with other entries
+# before it, in the middle of the string, or by itself.
+feature_gates = feature_gates.replace('RemoveSelfLink=false,', '')
+feature_gates = feature_gates.replace(',RemoveSelfLink=false', '')
+feature_gates = feature_gates.replace('RemoveSelfLink=false', '')
+
+if not feature_gates:
+    # No feature gates left, so delete the entry
+    syslog.syslog('Deleting kube-apiserver feature gates.')
+    info['apiserver_extra_args'].pop('feature-gates', None)
+else:
+    # Update the feature gates with the new value
+    syslog.syslog('Modifying kube-apiserver feature gates.')
+    info['apiserver_extra_args']['feature-gates'] = feature_gates
+
+# Write out the new file.
+try:
+    with open(FILENAME, 'w') as outfile:
+        yaml.safe_dump(info, outfile, default_flow_style=False, sort_keys=False)
+except Exception as exc:
+    syslog.syslog(syslog.LOG_ERR, "Problem writing to {}".format(FILENAME))
+    syslog.syslog(syslog.LOG_ERR, str(exc))
+    exit(1)
+
+syslog.syslog('Successfully wrote file with RemoveSelfLink=false removed.')

kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh

@@ -67,6 +67,75 @@ function get_kubeadm_configmap {
     fi
 }
 
+# Remove the "RemoveSelfLink=false" kube-apiserver feature gate from the service parameters.
+# This is needed to ensure that a backup taken after the upgrade to K8s 1.24 will be
+# properly restored without this feature gate.  (K8s 1.24 no longer supports this feature gate.)
+function update_feature_gates_service_param_v1_24 {
+
+    # Enable authentication.
+    source /etc/platform/openrc
+
+    # Check if any kube-apiserver feature gates are specified in the service parameters.
+    TMP=`system service-parameter-list|grep kube_apiserver|grep feature-gates`
+    if [ $? -ne 0 ]; then
+        # no feature-gates specified, nothing to do
+        LOG "No kube-apiserver feature-gates service param specified, nothing to do."
+        return
+    fi
+
+    # Get the actual feature gates.
+    # The xargs call here strips any whitespace.
+    FEATURE_GATES=`echo ${TMP}|cut -d'|' -f6|xargs`
+    if [ -z "${FEATURE_GATES}" ]; then
+        # No feature-gates, nothing to do.  Really shouldn't hit this, just being paranoid.
+        LOG "No kube-apiserver feature-gates specified, nothing to do."
+        return
+    fi
+
+    # Check if the specific feature gate we care about is specified.
+    echo ${FEATURE_GATES}|grep -q "RemoveSelfLink=false"
+    if [ $? -ne 0 ]; then
+        # RemoveSelfLink=false is not specified, nothing to do
+        LOG "RemoveSelfLink=false kube-apiserver feature gate not specified, nothing to do."
+        return
+    fi
+
+    # Remove "RemoveSelfLink=false" from the feature gates.
+    # We need to handle the case where it could be at the beginning of the string
+    # with other entries after it, or at the end of the string with other entries
+    # before it, in the middle of the string, or by itself.
+    NEW_FEATURE_GATES=${FEATURE_GATES//'RemoveSelfLink=false,'/}
+    NEW_FEATURE_GATES=${NEW_FEATURE_GATES//',RemoveSelfLink=false'/}
+    NEW_FEATURE_GATES=${NEW_FEATURE_GATES//'RemoveSelfLink=false'/}
+
+    if [ -z "${NEW_FEATURE_GATES}" ]; then
+        # no other feature gates, so delete the service parameter rather than modify it
+        UUID=`echo ${TMP}|cut -d'|' -f2`
+        system service-parameter-delete ${UUID}
+        if [ $? -eq 0 ]; then
+            LOG "Successfully deleted RemoveSelfLink=false feature gate service parameter ${UUID}"
+        else
+            LOG "Failed to delete RemoveSelfLink=false feature gate service parameter ${UUID}"
+        fi
+    else
+        # need to modify the service parameter with the new feature gates
+        system service-parameter-modify kubernetes kube_apiserver feature-gates=${NEW_FEATURE_GATES}
+        if [ $? -eq 0 ]; then
+            LOG "Successfully modified kube-apiserver feature gate service parameter."
+        else
+            LOG "Failed to modify kube-apiserver feature gate service parameter."
+        fi
+    fi
+}
+
+# This needs to edit a yaml file, so we call out to a python helper.
+function update_kube_extra_config_bootstrap_v1_24 {
+    /usr/local/sbin/sanitize_feature_gates.py
+    if [ $? -ne 0 ]; then
+        LOG "Problem running sanitize_feature_gates.py, may cause problems with backup/restore."
+    fi
+}
+
 # Update feature gates for version 1.24
 function update_feature_gates_v1_24 {
 
@@ -156,6 +225,8 @@ until  [ ${counter} -gt ${RETRIES} ]; do
     if [[ "${K8S_VERSION}" == "v1.21.8" ]]; then
         update_feature_gates_v1_22
     elif [[ "${K8S_VERSION}" == "v1.23.1" ]]; then
+        update_feature_gates_service_param_v1_24
+        update_kube_extra_config_bootstrap_v1_24
         update_feature_gates_v1_24
     else
         LOG "No update required for kubeadm configmap"