From 6e7736059ac927860951c876ad2693c03d9bd51b Mon Sep 17 00:00:00 2001 From: Kaustubh Dhokte Date: Mon, 14 Nov 2022 16:28:47 +0000 Subject: [PATCH] update feature-gates for specific k8s version This change modifies upgrade_k8s_config.sh to support updating k8s feature-gates for different k8s versions. With every k8s release, default values of some feature-gate are changed and usage of some feature-gates often gets deprecated. The script runs during each k8s control plane upgrade before upgrading first master. It modifies kubeadm-config configmap with features-gates as required for the specific k8s version we are upgrading to. The set of changes here, https://opendev.org/starlingx/integ/commit/a6a5349d025487672fe818aae36a2020a9f9f08c (k8s-1.22.5: remove feature-gates), https://opendev.org/starlingx/stx-puppet/commit/1cdfd7828622b88318058ae4b13b629fc1d94be9 (Add a puppet class to support k8s feature-gate update), and https://opendev.org/starlingx/config/commit/cc3cdbd6474e574a20bc71b35cb6875e359875aa (apply feature-gate update during upgrade-activate) were added for stx 6.0 to stx 7.0 upgrade (CentOS) for changes in feature-gates with respect to k8s 1.22. Instead of adding that script to Debian and maintaining two different scripts, going forward we can maintain this single script to accommodate any change in feature-gates (or any other config in kubeadm-config) with respect to the specific k8s version we are upgrading to. Test Plan: PASS: K8s upgrade 1.21.8 to 1.22.5 PASS: k8s upgrade 1.23.1 to 1.24.4 PASS: shellcheck run PASS: replace_configmap function was unit tested separately. Closes-Bug: 1996546 Closes-Bug: 1990880 Signed-off-by: Kaustubh Dhokte Change-Id: Ib693d7892aee2da91d612789b64ff38a65da5ccb --- .../debian/deb_folder/upgrade_k8s_config.sh | 141 ++++++++++++++---- 1 file changed, 108 insertions(+), 33 deletions(-) diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh b/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh index 882aa2c65..61734af3f 100755 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh @@ -4,11 +4,22 @@ # SPDX-License-Identifier: Apache-2.0 # -# This will run during a k8s upgrade as a part of the control-plane upgrade of +# This will run during k8s upgrades as a part of the control-plane upgrade of # the first master. It updates the kubeadm-config configmap to edit the -# manifests and remove the 'feature-gates' lines. +# manifests and updates feature-gates as required for the specific k8s version +# it is upgrading to. +# +# The script updates feature gates for versions 1.22 and 1.24. +# +# It removes below feature gates from kube-apiserver configmap while upgrading +# k8s 1.21.8 to 1.22.5: SCTPSupport=true, HugePageStorageMediumSize=true +# and TTLAfterFinished=true and removes RemoveSelfLink=false while upgrading +# 1.23.1 to 1.24.4. # # Background: +# HugePageStorageMediumSize is deprecated in Kubernetes 1.22 +# SCTPSupport blocks kube-apiserver pod to spawn after control-plane upgrade. +# TTLAfterFinished value defaults to true from k8s 1.21 # Kubernetes 1.24 no longer allows setting kube-apsierver feature-gate # RemoveSelfLink=false. All the other feature gates we were using now default # to true so we don't want to specify them anymore. @@ -28,34 +39,36 @@ function ERROR { # Cleanup and exit function cleanup_and_exit { - rm -v -f "${KUBEADM_CONFIGMAP_TMPFILE}" + rm -v -f "${KUBEADM_CONFIGMAP_TMPFILE}" 2>/dev/null exit "${1:-0}" } -# Update the configmap for kubeadm -function update_apiserver_configmap { +function get_kubeadm_configmap { LOG "Retrieving kubeadm configmap: ${KUBEADM_CONFIGMAP_TMPFILE}" - counter=0 - RETRIES=10 + local counter=0 + local RETRIES=10 RC=0 - until [ $counter -gt $RETRIES ]; do + until [ ${counter} -gt ${RETRIES} ]; do kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system get \ configmap kubeadm-config -o yaml > "${KUBEADM_CONFIGMAP_TMPFILE}" RC=$? - if [ "$RC" = "0" ] ; then + if [ "${RC}" == "0" ] ; then LOG "Kubeadm configmap retrieved." break - ((counter+=1)) fi - ERROR "Failed to retrieve kubeadm configmap, retrying..." + ERROR "Error retrieving kubeadm configmap, retrying..." sleep 5 - ((counter+=1)) + counter=$(( counter+1 )) done - if [ $counter -gt $RETRIES ]; then - ERROR "Failed to retrieve kubeadm configmap with error code [$RC]". - cleanup_and_exit $RC + if [ ${counter} -gt ${RETRIES} ]; then + ERROR "Failed to retrieve kubeadm configmap with error code [${RC}]". + cleanup_and_exit ${RC} fi +} + +# Update feature gates for version 1.24 +function update_feature_gates_v1_24 { if grep -q 'RemoveSelfLink=false' "${KUBEADM_CONFIGMAP_TMPFILE}"; then LOG "Updating kube-apiserver feature-gates in retrieved kubeadm-config" @@ -63,36 +76,98 @@ function update_apiserver_configmap { if ! grep -q 'RemoveSelfLink=false' "${KUBEADM_CONFIGMAP_TMPFILE}"; then LOG "Successfully updated retrieved kubeadm-config" - if kubectl --kubeconfig=/etc/kubernetes/admin.conf replace -f \ - "${KUBEADM_CONFIGMAP_TMPFILE}"; then - LOG 'Successfully replaced updated kubeadm configmap.' - else - RC=$? - ERROR "Failed to replace updated kubeadm configmap with error code: [$RC]" - cleanup_and_exit $RC - fi else ERROR 'Failed to update kube-apiserver feature-gates with an unknown error' cleanup_and_exit 1 fi else RC=$? - ERROR "Failed to update ${KUBEADM_CONFIGMAP_TMPFILE} with error code: [$RC]" - cleanup_and_exit $RC + ERROR "Failed to update ${KUBEADM_CONFIGMAP_TMPFILE} with error code: [${RC}]" + cleanup_and_exit ${RC} fi else LOG "Kubeadm configmap was already updated with RemoveSelfLink=false removed. Nothing to do." fi } -# Update kube-apiserver configMap only for k8s 1.23.1 -K8S_VERSION_FROM='v1.23.1' +# Update feature gates for version 1.22 +function update_feature_gates_v1_22 { + LOG "Updating kube-apiserver feature-gates in retrieved kubeadm-config" + + # Update api-server feature-gates + sed -i \ + 's/^\( *\)feature-gates:\s.*RemoveSelfLink=false/\1feature-gates: RemoveSelfLink=false/g' \ + "${KUBEADM_CONFIGMAP_TMPFILE}" + RC=$? + if [ "${RC}" == "0" ]; then + LOG "Successfully updated kube-apiserver feature-gates in retrieved kubeadm-config" + else + ERROR "Failed to update kube-apiserver feature-gates in retrieved kubeadm-config with error code: [${RC}]" + cleanup_and_exit ${RC} + fi + + # update controller-manager feature-gates + sed -i \ + '/feature-gates: TTLAfterFinished=true/d' "${KUBEADM_CONFIGMAP_TMPFILE}" + RC=$? + if [ "${RC}" == "0" ]; then + LOG "Successfully updated controller-manager feature-gates in retrieved kubeadm-config" + else + # we need not gracefully exit here as failing to update this does not + # make any difference to the k8s cluster functions as default value of + # TTLAfterFinished is true + ERROR "Failed to update controller-manager feature-gates in retrieved kubeadm-config with error code: [${RC}]" + fi +} + +function replace_configmap { + + output=$(kubectl --kubeconfig=/etc/kubernetes/admin.conf replace -f "${KUBEADM_CONFIGMAP_TMPFILE}" 2>&1) + RC=$? + if [ "${RC}" == "0" ]; then + LOG 'Successfully replaced updated kubeadm configmap.' + else + # LP-1996546. kubectl replace expects replacing object to be latest version. + # Although there is low chance that kubeadm-configmap is modified by other + # process in between calls get_kubeadm_configmap and replace_configmap, + # we should still check for that error. If it is the case, then we retry + # modifying and replacing the newer version. + if [[ ${output} == *"the object has been modified; please apply your changes to the latest version and try again" ]]; then + LOG "kubeadm configmap is not the newest version." + else + ERROR "Failed to replace updated kubeadm configmap with error code: [${RC}]" + cleanup_and_exit ${RC} + fi + fi + return ${RC} +} + K8S_VERSION=$(kubectl version --output=yaml| grep -m1 -oP 'gitVersion: \K(\S+)') -if [[ "${K8S_VERSION}" == "${K8S_VERSION_FROM}" ]]; then - LOG "k8s version ${K8S_VERSION} matches ${K8S_VERSION_FROM}" - update_apiserver_configmap -else - LOG "k8s version ${K8S_VERSION} does not match ${K8S_VERSION_FROM}, skip update" -fi +LOG "k8s version: ${K8S_VERSION}" +counter=0 +RETRIES=3 +# Most errors during script execution result in exiting the script except one error. +# If kubeadm-configmap is modified by external process after it is +# retrieved in function get_kubeadm_configmap and before it is modified +# and replaced in the function replace_configmap, we should retry modifying +# and replacing the latest version. +until [ ${counter} -gt ${RETRIES} ]; do + get_kubeadm_configmap + if [[ "${K8S_VERSION}" == "v1.21.8" ]]; then + update_feature_gates_v1_22 + elif [[ "${K8S_VERSION}" == "v1.23.1" ]]; then + update_feature_gates_v1_24 + else + LOG "No update required for kubeadm configmap" + break + fi + replace_configmap + if [ "$?" == "0" ]; then + break + else + LOG "Retrying to update the configmap..." + counter=$(( counter+1 )) + fi +done cleanup_and_exit 0