clean up feature gates on k8s upgrade

During a K8s feature upgrade from 1.23 to 1.24 we need to remove
the "RemoveSelfLink=false" feature gate from kube-apiserver.

We had previously handled updating the kubeadm configmap, which
was sufficient to handle the running system.  However, in order
to properly handle backup and restore after the K8s upgrade to
1.24 (and just for general tidiness) we need to also remove the
feature gate from the saved service parameters and from the
last_kube_extra_config_bootstrap.yaml file.

It's possible that there are other kube-apiserver feature gates
specified by the end user, this adds a bit of complexity to the
code.

Test Plan:
PASS: Test python script and bash script in isolation.
PASS: End-to-end test with k8s upgrade and backup/restore with
      manual modification of service parameters and yaml file.
      Tested with AIO-DX, AIO-SX unoptimised restore, and
      AIO-SX optimised restore.
PASS: K8s upgrade using the new code, ensure service parameter
      and last_kube_extra_config_bootstrap.yaml have been
      updated with "RemoveSelfLink=false" feature gate removed.

Closes-Bug: 1999095
Signed-off-by: Chris Friesen <chris.friesen@windriver.com>
Change-Id: I82ecd821d4e1745ab0f480f9f9c0178757521038

kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install

@@ -8,3 +8,4 @@ etc/systemd/system.conf.d/kubernetes-accounting.conf
 usr/lib/tmpfiles.d/kubernetes.conf
 usr/local/sbin/sanitize_kubelet_reserved_cpus.sh
 usr/local/sbin/upgrade_k8s_config.sh
+usr/local/sbin/sanitize_feature_gates.py

kubernetes/kubernetes-unversioned/debian/deb_folder/rules

@@ -66,6 +66,7 @@ override_dh_install:
 	install -v -m 0700 -d ${DEBIAN_DESTDIR}${_local_sbindir}
 	install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/sanitize_kubelet_reserved_cpus.sh
 	install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/upgrade_k8s_config.sh
+	install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/sanitize_feature_gates.py
 
 	dh_install
 

kubernetes/kubernetes-unversioned/debian/deb_folder/sanitize_feature_gates.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+'''Removes the "RemoveSelfLink=false" kube-apiserver feature gate
+
+Usage:
+    ./sanitize_feature_gates.py
+
+This script is just to remove the "RemoveSelfLink=false" kube-apiserver
+feature gate from the last_kube_extra_config_bootstrap.yaml file when
+doing an upgrade from K8s 1.23 to 1.24.
+Once we no longer need to worry about upgrading from 1.23 we can remove this.
+
+Copyright (c) 2022 Wind River Systems, Inc.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at:
+      http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software  distributed
+under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+OR CONDITIONS OF ANY KIND, either express or implied. 
+'''
+
+import syslog
+import yaml
+
+FILENAME = "/opt/platform/config/22.12/last_kube_extra_config_bootstrap.yaml"
+
+# The above file contains information that is used during backup and restore.
+# We want to remove the "RemoveSelfLink=false" kube-apiserver feature gate
+# if it's present so that if we do a backup and restore after the upgrade
+# to K8s 1.24 we won't try to use this feature gate any more.  Any other feature
+# gates should be left alone, and if there aren't any other feature gates
+# we want to delete the entire "feature-gates" entry.
+
+# Logs will go to /var/log/daemon.log
+syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_DAEMON)
+
+try:
+    with open(FILENAME, "r") as stream:
+        info = yaml.safe_load(stream)
+except Exception as exc:
+    syslog.syslog(syslog.LOG_ERR, "Problem reading from {}".format(FILENAME))
+    syslog.syslog(syslog.LOG_ERR, str(exc))
+    exit(1)
+
+try:
+    feature_gates = info['apiserver_extra_args']['feature-gates']
+except KeyError:
+    # No apiserver feature gates, nothing to do
+    syslog.syslog('No kube-apiserver feature gates, nothing to do.')
+    exit(0)
+
+if "RemoveSelfLink=false" not in feature_gates:
+    # Nothing to do
+    syslog.syslog('No changes needed in kube-apiserver feature gates.')
+    exit(0)
+
+# Remove "RemoveSelfLink=false" from the feature gates
+# we need to handle the case where it could be at the beginning of the string
+# with other entries after it, or at the end of the string with other entries
+# before it, in the middle of the string, or by itself.
+feature_gates = feature_gates.replace('RemoveSelfLink=false,', '')
+feature_gates = feature_gates.replace(',RemoveSelfLink=false', '')
+feature_gates = feature_gates.replace('RemoveSelfLink=false', '')
+
+if not feature_gates:
+    # No feature gates left, so delete the entry
+    syslog.syslog('Deleting kube-apiserver feature gates.')
+    info['apiserver_extra_args'].pop('feature-gates', None)
+else:
+    # Update the feature gates with the new value
+    syslog.syslog('Modifying kube-apiserver feature gates.')
+    info['apiserver_extra_args']['feature-gates'] = feature_gates
+
+# Write out the new file.
+try:
+    with open(FILENAME, 'w') as outfile:
+        yaml.safe_dump(info, outfile, default_flow_style=False, sort_keys=False)
+except Exception as exc:
+    syslog.syslog(syslog.LOG_ERR, "Problem writing to {}".format(FILENAME))
+    syslog.syslog(syslog.LOG_ERR, str(exc))
+    exit(1)
+
+syslog.syslog('Successfully wrote file with RemoveSelfLink=false removed.')

kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh

@@ -67,6 +67,75 @@ function get_kubeadm_configmap {
     fi
 }
 
+# Remove the "RemoveSelfLink=false" kube-apiserver feature gate from the service parameters.
+# This is needed to ensure that a backup taken after the upgrade to K8s 1.24 will be
+# properly restored without this feature gate.  (K8s 1.24 no longer supports this feature gate.)
+function update_feature_gates_service_param_v1_24 {
+
+    # Enable authentication.
+    source /etc/platform/openrc
+
+    # Check if any kube-apiserver feature gates are specified in the service parameters.
+    TMP=`system service-parameter-list|grep kube_apiserver|grep feature-gates`
+    if [ $? -ne 0 ]; then
+        # no feature-gates specified, nothing to do
+        LOG "No kube-apiserver feature-gates service param specified, nothing to do."
+        return
+    fi
+
+    # Get the actual feature gates.
+    # The xargs call here strips any whitespace.
+    FEATURE_GATES=`echo ${TMP}|cut -d'|' -f6|xargs`
+    if [ -z "${FEATURE_GATES}" ]; then
+        # No feature-gates, nothing to do.  Really shouldn't hit this, just being paranoid.
+        LOG "No kube-apiserver feature-gates specified, nothing to do."
+        return
+    fi
+
+    # Check if the specific feature gate we care about is specified.
+    echo ${FEATURE_GATES}|grep -q "RemoveSelfLink=false"
+    if [ $? -ne 0 ]; then
+        # RemoveSelfLink=false is not specified, nothing to do
+        LOG "RemoveSelfLink=false kube-apiserver feature gate not specified, nothing to do."
+        return
+    fi
+
+    # Remove "RemoveSelfLink=false" from the feature gates.
+    # We need to handle the case where it could be at the beginning of the string
+    # with other entries after it, or at the end of the string with other entries
+    # before it, in the middle of the string, or by itself.
+    NEW_FEATURE_GATES=${FEATURE_GATES//'RemoveSelfLink=false,'/}
+    NEW_FEATURE_GATES=${NEW_FEATURE_GATES//',RemoveSelfLink=false'/}
+    NEW_FEATURE_GATES=${NEW_FEATURE_GATES//'RemoveSelfLink=false'/}
+
+    if [ -z "${NEW_FEATURE_GATES}" ]; then
+        # no other feature gates, so delete the service parameter rather than modify it
+        UUID=`echo ${TMP}|cut -d'|' -f2`
+        system service-parameter-delete ${UUID}
+        if [ $? -eq 0 ]; then
+            LOG "Successfully deleted RemoveSelfLink=false feature gate service parameter ${UUID}"
+        else
+            LOG "Failed to delete RemoveSelfLink=false feature gate service parameter ${UUID}"
+        fi
+    else
+        # need to modify the service parameter with the new feature gates
+        system service-parameter-modify kubernetes kube_apiserver feature-gates=${NEW_FEATURE_GATES}
+        if [ $? -eq 0 ]; then
+            LOG "Successfully modified kube-apiserver feature gate service parameter."
+        else
+            LOG "Failed to modify kube-apiserver feature gate service parameter."
+        fi
+    fi
+}
+
+# This needs to edit a yaml file, so we call out to a python helper.
+function update_kube_extra_config_bootstrap_v1_24 {
+    /usr/local/sbin/sanitize_feature_gates.py
+    if [ $? -ne 0 ]; then
+        LOG "Problem running sanitize_feature_gates.py, may cause problems with backup/restore."
+    fi
+}
+
 # Update feature gates for version 1.24
 function update_feature_gates_v1_24 {
 
@@ -156,6 +225,8 @@ until  [ ${counter} -gt ${RETRIES} ]; do
     if [[ "${K8S_VERSION}" == "v1.21.8" ]]; then
         update_feature_gates_v1_22
     elif [[ "${K8S_VERSION}" == "v1.23.1" ]]; then
+        update_feature_gates_service_param_v1_24
+        update_kube_extra_config_bootstrap_v1_24
         update_feature_gates_v1_24
     else
         LOG "No update required for kubeadm configmap"