From 188a92fe0ed7c4b2f6d7ac04c3ddecdbf86efe53 Mon Sep 17 00:00:00 2001 From: Jim Gauld Date: Thu, 27 Oct 2022 12:06:28 -0400 Subject: [PATCH] Debian: Add kubernetes 1.23.1 upgrade script to kubernetes-unversioned Add two files to Debian package kubernetes-unversioned to support kubernetes upgrades. We need to remove undesired feature gates from the kubeadm configmap before the upgrade to 1.24. The following two files are both required by puppet class: platform::kubernetes::upgrade_first_control_plane, * upgrade_k8s_config.sh - upgrade script to update apiserver/kubelet configmap * kubelet_override.yaml - empty / non-customized override file. This file is being referenced, we keep override functionality even though we do not currently require kubelet overrides. Test Plan: TODO: Kubernetes upgrade from k8s 1.23 to 1.24 TODO: Platform upgrade from k8s 1.23, followed by kubernetes upgrade from k8s 1.23 to 1.24 Story: 2010301 Task: 46692 Signed-off-by: Jim Gauld Change-Id: Ibb2438c79b2983d2bc6beeaec287795f3c6c124f --- .../debian/deb_folder/copyright | 2 +- .../debian/deb_folder/kubelet_override.yaml | 2 + .../deb_folder/kubernetes-unversioned.install | 2 + .../debian/deb_folder/rules | 2 + .../debian/deb_folder/upgrade_k8s_config.sh | 98 +++++++++++++++++++ 5 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 kubernetes/kubernetes-unversioned/debian/deb_folder/kubelet_override.yaml create mode 100755 kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/copyright b/kubernetes/kubernetes-unversioned/debian/deb_folder/copyright index bc38cc017..51b507756 100644 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/copyright +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/copyright @@ -3,7 +3,7 @@ Upstream-Name: kubernetes-unversioned Source: https://opendev.org/starlingx/integ/src/branch/master/kubernetes/kubernetes-unversioned Files: * -Copyright: (c) 2021 Wind River Systems, Inc +Copyright: (c) 2022 Wind River Systems, Inc (c) Others (See individual files for more details) License: Apache-2 Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/kubelet_override.yaml b/kubernetes/kubernetes-unversioned/debian/deb_folder/kubelet_override.yaml new file mode 100644 index 000000000..b9fd0fe04 --- /dev/null +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/kubelet_override.yaml @@ -0,0 +1,2 @@ +--- +# no customizations diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install b/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install index 717ad0279..0095bb10a 100644 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install @@ -2,7 +2,9 @@ usr/lib/systemd/system/kubelet.service etc/kubernetes/config etc/kubernetes/kubelet etc/kubernetes/kubelet.kubeconfig +etc/kubernetes/kubelet_override.yaml etc/kubernetes/proxy etc/systemd/system.conf.d/kubernetes-accounting.conf usr/lib/tmpfiles.d/kubernetes.conf usr/local/sbin/sanitize_kubelet_reserved_cpus.sh +usr/local/sbin/upgrade_k8s_config.sh diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/rules b/kubernetes/kubernetes-unversioned/debian/deb_folder/rules index d6ad1eb12..8ab58fb08 100755 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/rules +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/rules @@ -43,6 +43,7 @@ override_dh_install: install -v -m 644 -t ${DEBIAN_DESTDIR}/etc/${_k8s_name} contrib/init/systemd/environ/kubelet install -v -m 644 -t ${DEBIAN_DESTDIR}/etc/${_k8s_name} contrib/init/systemd/environ/kubelet.kubeconfig install -v -m 644 -t ${DEBIAN_DESTDIR}/etc/${_k8s_name} contrib/init/systemd/environ/proxy + install -v -m 644 -t ${DEBIAN_DESTDIR}/etc/${_k8s_name} debian/kubelet_override.yaml # install config files install -v -d -m 0755 ${DEBIAN_DESTDIR}/usr/lib/tmpfiles.d @@ -64,6 +65,7 @@ override_dh_install: # install scripts install -v -m 0700 -d ${DEBIAN_DESTDIR}${_local_sbindir} install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/sanitize_kubelet_reserved_cpus.sh + install -v -m 0700 -t ${DEBIAN_DESTDIR}${_local_sbindir} debian/upgrade_k8s_config.sh dh_install diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh b/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh new file mode 100755 index 000000000..882aa2c65 --- /dev/null +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/upgrade_k8s_config.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# Copyright (c) 2022 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# This will run during a k8s upgrade as a part of the control-plane upgrade of +# the first master. It updates the kubeadm-config configmap to edit the +# manifests and remove the 'feature-gates' lines. +# +# Background: +# Kubernetes 1.24 no longer allows setting kube-apsierver feature-gate +# RemoveSelfLink=false. All the other feature gates we were using now default +# to true so we don't want to specify them anymore. + +# Temporary configuration file +KUBEADM_CONFIGMAP_TMPFILE=$(mktemp /tmp/kubeadm_cm.yaml.XXXXXX 2>/dev/null) + +# Log info message to /var/log/daemon.log +function LOG { + logger -p daemon.info "$0($$): " "${@}" +} + +# Log error message to /var/log/daemon.log +function ERROR { + logger -s -p daemon.error "$0($$): " "${@}" +} + +# Cleanup and exit +function cleanup_and_exit { + rm -v -f "${KUBEADM_CONFIGMAP_TMPFILE}" + exit "${1:-0}" +} + +# Update the configmap for kubeadm +function update_apiserver_configmap { + LOG "Retrieving kubeadm configmap: ${KUBEADM_CONFIGMAP_TMPFILE}" + counter=0 + RETRIES=10 + RC=0 + until [ $counter -gt $RETRIES ]; do + kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system get \ + configmap kubeadm-config -o yaml > "${KUBEADM_CONFIGMAP_TMPFILE}" + RC=$? + if [ "$RC" = "0" ] ; then + LOG "Kubeadm configmap retrieved." + break + ((counter+=1)) + fi + ERROR "Failed to retrieve kubeadm configmap, retrying..." + sleep 5 + ((counter+=1)) + done + + if [ $counter -gt $RETRIES ]; then + ERROR "Failed to retrieve kubeadm configmap with error code [$RC]". + cleanup_and_exit $RC + fi + + if grep -q 'RemoveSelfLink=false' "${KUBEADM_CONFIGMAP_TMPFILE}"; then + LOG "Updating kube-apiserver feature-gates in retrieved kubeadm-config" + if sed -i '/feature-gates/d' "${KUBEADM_CONFIGMAP_TMPFILE}"; then + if ! grep -q 'RemoveSelfLink=false' "${KUBEADM_CONFIGMAP_TMPFILE}"; + then + LOG "Successfully updated retrieved kubeadm-config" + if kubectl --kubeconfig=/etc/kubernetes/admin.conf replace -f \ + "${KUBEADM_CONFIGMAP_TMPFILE}"; then + LOG 'Successfully replaced updated kubeadm configmap.' + else + RC=$? + ERROR "Failed to replace updated kubeadm configmap with error code: [$RC]" + cleanup_and_exit $RC + fi + else + ERROR 'Failed to update kube-apiserver feature-gates with an unknown error' + cleanup_and_exit 1 + fi + else + RC=$? + ERROR "Failed to update ${KUBEADM_CONFIGMAP_TMPFILE} with error code: [$RC]" + cleanup_and_exit $RC + fi + else + LOG "Kubeadm configmap was already updated with RemoveSelfLink=false removed. Nothing to do." + fi +} + +# Update kube-apiserver configMap only for k8s 1.23.1 +K8S_VERSION_FROM='v1.23.1' +K8S_VERSION=$(kubectl version --output=yaml| grep -m1 -oP 'gitVersion: \K(\S+)') +if [[ "${K8S_VERSION}" == "${K8S_VERSION_FROM}" ]]; then + LOG "k8s version ${K8S_VERSION} matches ${K8S_VERSION_FROM}" + update_apiserver_configmap +else + LOG "k8s version ${K8S_VERSION} does not match ${K8S_VERSION_FROM}, skip update" +fi + +cleanup_and_exit 0