starlingx/integ
Code Issues Proposed changes

Remove broken/stale K8s upgrade script

Browse Source 
upgrade_k8s_config.sh does not support
"flow" style of YAML.  This results in
the configmap becoming corrupted when
upgrading from k8s 1.21 to 1.22.

Removing the legacy code, the changes to the
config map are now handled in sysinv.

Test Plan
PASS:
- Manually update scripts on controllers and worker nodes based on
  https://review.opendev.org/c/starlingx/stx-puppet/+/880547?tab=comments
- Perform manual upgrade from k8s v1.21.8 to v1.22.5
- Verify kubernetes successfully upgraded to v1.22.5

Test was performed in the lab with local changes
to verify the code.
Patch was not tested.

Closes-Bug: 2017696
Depends-On: https://review.opendev.org/c/starlingx/stx-puppet/+/880547
Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com>
Change-Id: Ic47c3e40418898917934309f69b35ad1f401eb06
This commit is contained in:
Gleb Aronsky 2023-04-13 14:34:07 -07:00
parent 56e5360de1
commit 653f65106e
5 changed files with 0 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
kubernetes/kubernetes-unversioned/centos/files/upgrade_k8s_config.sh
View File

@ -1,85 +0,0 @@
#!/bin/bash
# Copyright (c) 2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This will run for every k8s upgrade as a part of the control-plane upgrade of the first master.
# - updates kubeadm-config configmap to configure kube-apiserver manifest with RemoveSelfLink=false.
# - generates a kubelet config override file to configure cgroupDriver=cgroupfs.
# This is consumed by kubeadm upgrade apply
#
# Background:
# Kubernetes 1.21 changed cgroupDriver default to systemd (was cgroupfs).
# Kubernetes 1.20 changed feature-gates RemoveSelfLink default to true.
KUBEADM_CONFIGMAP_TMPFILE='/tmp/kubeadm_cm.yaml'
function log {
logger -p local1.info "$1"
}
# Update the configmap for kubeadm
function update_apiserver_configmap {
log "Retrieving kubeadm configmap to temporary location: ${KUBEADM_CONFIGMAP_TMPFILE}"
counter=0
RC=0
RETRIES=10
until [ $counter -gt $RETRIES ]; do
kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system get \
configmap kubeadm-config -o yaml > ${KUBEADM_CONFIGMAP_TMPFILE}
RC=$?
if [ "$RC" = "0" ] ; then
log "Kubeadm configmap retrieved."
break
fi
log "Error retrieving kubeadm configmap, retrying..."
sleep 5
let "counter+=1"
done
if [ $counter -gt $RETRIES ]; then
log "Failed to retrieve kubeadm configmap with error code [$RC]".
exit $RC
fi
if ! grep -q 'RemoveSelfLink=false' ${KUBEADM_CONFIGMAP_TMPFILE}; then
log "Updating kube-apiserver feature-gates in retrieved kubeadm-config"
if sed -i \
'/^\s*feature-gates:\s*.*HugePageStorageMediumSize='\
'true/ s/$/,RemoveSelfLink=false/' ${KUBEADM_CONFIGMAP_TMPFILE}; then
if grep -q 'RemoveSelfLink=false' ${KUBEADM_CONFIGMAP_TMPFILE};
then
log "Successfully updated retrieved kubeadm-config"
if kubectl --kubeconfig=/etc/kubernetes/admin.conf replace -f \
${KUBEADM_CONFIGMAP_TMPFILE}; then
log 'Successfully replaced updated kubeadm configmap.'
else
RC=$?
log "Failed to replace updated kubeadm configmap with error code: [$RC]"
exit $RC
fi
else
log 'Failed to update kube-apiserver feature-gates with an unknown error'
exit -1
fi
else
RC=$?
log "Failed to update ${KUBEADM_CONFIGMAP_TMPFILE} with error code: [$RC]"
exit $RC
fi
else
log "Kubeadm configmap was already updated with RemoveSelfLink=false. Nothing to do."
fi
rm -f ${KUBEADM_CONFIGMAP_TMPFILE}
}
update_apiserver_configmap
exit 0

5
kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec
View File

@ -39,8 +39,6 @@ Source2: kubernetes-accounting.conf
# kubelet config overrides parameters
Source3: kubelet_override.yaml
Source4: upgrade_k8s_config.sh
Source5: sanitize_kubelet_reserved_cpus.sh
Patch1: kubelet-service-remove-docker-dependency.patch
@ -101,8 +99,6 @@ install -p -D -m 644 %{SOURCE3} %{buildroot}%{_sysconfdir}/%{_k8s_name}/kubelet_
install -d %{buildroot}%{local_sbindir}
# install execution scripts
install -m 700 %{SOURCE4} %{buildroot}/%{local_sbindir}/upgrade_k8s_config.sh
install -m 700 %{SOURCE5} %{buildroot}/%{local_sbindir}/sanitize_kubelet_reserved_cpus.sh
# install service files
@ -123,7 +119,6 @@ install -v -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system.conf.d %{SOU
%dir %{_curr_stage2}
# the following are execution scripts
%{local_sbindir}/upgrade_k8s_config.sh
%{local_sbindir}/sanitize_kubelet_reserved_cpus.sh
# the following are symlinks

1
kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install
View File

@ -7,4 +7,3 @@ etc/kubernetes/proxy
etc/systemd/system.conf.d/kubernetes-accounting.conf
usr/lib/tmpfiles.d/kubernetes.conf
usr/local/sbin/sanitize_kubelet_reserved_cpus.sh
usr/local/sbin/upgrade_k8s_config.sh

1
kubernetes/kubernetes-unversioned/debian/deb_folder/rules
View File

@ -65,7 +65,6 @@ override_dh_install:
# install scripts
install -v -m 0700 -d ${DEBIAN_DESTDIR}${_local_sbindir}
install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/sanitize_kubelet_reserved_cpus.sh
install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/upgrade_k8s_config.sh
dh_install

148
kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh
View File

@ -1,148 +0,0 @@
#!/bin/bash
# Copyright (c) 2022 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This will run during k8s upgrades as a part of the control-plane upgrade of
# the first master. It updates the kubeadm-config configmap to edit the
# manifests and updates feature-gates as required for the specific k8s version
# it is upgrading to.
#
# The script updates feature gates for versions 1.22 and 1.24.
#
# It removes below feature gates from kube-apiserver configmap while upgrading
# k8s 1.21.8 to 1.22.5: SCTPSupport=true, HugePageStorageMediumSize=true
# and TTLAfterFinished=true and removes RemoveSelfLink=false while upgrading
# 1.23.1 to 1.24.4.
#
# Background:
# HugePageStorageMediumSize is deprecated in Kubernetes 1.22
# SCTPSupport blocks kube-apiserver pod to spawn after control-plane upgrade.
# TTLAfterFinished value defaults to true from k8s 1.21
# Kubernetes 1.24 no longer allows setting kube-apsierver feature-gate
# RemoveSelfLink=false. All the other feature gates we were using now default
# to true so we don't want to specify them anymore.
# Temporary configuration file
KUBEADM_CONFIGMAP_TMPFILE=$(mktemp /tmp/kubeadm_cm.yaml.XXXXXX 2>/dev/null)
# Log info message to /var/log/daemon.log
function LOG {
logger -p daemon.info "$0($$): " "${@}"
}
# Log error message to /var/log/daemon.log
function ERROR {
logger -s -p daemon.error "$0($$): " "${@}"
}
# Cleanup and exit
function cleanup_and_exit {
rm -v -f "${KUBEADM_CONFIGMAP_TMPFILE}" 2>/dev/null
exit "${1:-0}"
}
function get_kubeadm_configmap {
LOG "Retrieving kubeadm configmap: ${KUBEADM_CONFIGMAP_TMPFILE}"
local counter=0
local RETRIES=10
RC=0
until [ ${counter} -gt ${RETRIES} ]; do
kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system get \
configmap kubeadm-config -o yaml > "${KUBEADM_CONFIGMAP_TMPFILE}"
RC=$?
if [ "${RC}" == "0" ] ; then
LOG "Kubeadm configmap retrieved."
break
fi
ERROR "Error retrieving kubeadm configmap, retrying..."
sleep 5
counter=$(( counter+1 ))
done
if [ ${counter} -gt ${RETRIES} ]; then
ERROR "Failed to retrieve kubeadm configmap with error code [${RC}]".
cleanup_and_exit ${RC}
fi
}
# Update feature gates for version 1.22
function update_feature_gates_v1_22 {
LOG "Updating kube-apiserver feature-gates in retrieved kubeadm-config"
# Update api-server feature-gates
sed -i \
's/^\( *\)feature-gates:\s.*RemoveSelfLink=false/\1feature-gates: RemoveSelfLink=false/g' \
"${KUBEADM_CONFIGMAP_TMPFILE}"
RC=$?
if [ "${RC}" == "0" ]; then
LOG "Successfully updated kube-apiserver feature-gates in retrieved kubeadm-config"
else
ERROR "Failed to update kube-apiserver feature-gates in retrieved kubeadm-config with error code: [${RC}]"
cleanup_and_exit ${RC}
fi
# update controller-manager feature-gates
sed -i \
'/feature-gates: TTLAfterFinished=true/d' "${KUBEADM_CONFIGMAP_TMPFILE}"
RC=$?
if [ "${RC}" == "0" ]; then
LOG "Successfully updated controller-manager feature-gates in retrieved kubeadm-config"
else
# we need not gracefully exit here as failing to update this does not
# make any difference to the k8s cluster functions as default value of
# TTLAfterFinished is true
ERROR "Failed to update controller-manager feature-gates in retrieved kubeadm-config with error code: [${RC}]"
fi
}
function replace_configmap {
output=$(kubectl --kubeconfig=/etc/kubernetes/admin.conf replace -f "${KUBEADM_CONFIGMAP_TMPFILE}" 2>&1)
RC=$?
if [ "${RC}" == "0" ]; then
LOG 'Successfully replaced updated kubeadm configmap.'
else
# LP-1996546. kubectl replace expects replacing object to be latest version.
# Although there is low chance that kubeadm-configmap is modified by other
# process in between calls get_kubeadm_configmap and replace_configmap,
# we should still check for that error. If it is the case, then we retry
# modifying and replacing the newer version.
if [[ ${output} == *"the object has been modified; please apply your changes to the latest version and try again" ]]; then
LOG "kubeadm configmap is not the newest version."
else
ERROR "Failed to replace updated kubeadm configmap with error code: [${RC}]"
cleanup_and_exit ${RC}
fi
fi
return ${RC}
}
K8S_VERSION=$(kubectl version --output=yaml| grep -m1 -oP 'gitVersion: \K(\S+)')
LOG "k8s version: ${K8S_VERSION}"
counter=0
RETRIES=3
# Most errors during script execution result in exiting the script except one error.
# If kubeadm-configmap is modified by external process after it is
# retrieved in function get_kubeadm_configmap and before it is modified
# and replaced in the function replace_configmap, we should retry modifying
# and replacing the latest version.
until [ ${counter} -gt ${RETRIES} ]; do
get_kubeadm_configmap
if [[ "${K8S_VERSION}" == "v1.21.8" ]]; then
update_feature_gates_v1_22
else
LOG "No update required for kubeadm configmap"
break
fi
replace_configmap
if [ "$?" == "0" ]; then
break
else
LOG "Retrying to update the configmap..."
counter=$(( counter+1 ))
fi
done
cleanup_and_exit 0