diff --git a/kubernetes/kubernetes-1.18.1/centos/files/kubeadm.conf b/kubernetes/kubernetes-1.18.1/centos/files/kubeadm.conf index 2fb25ba27..da0611444 100644 --- a/kubernetes/kubernetes-1.18.1/centos/files/kubeadm.conf +++ b/kubernetes/kubernetes-1.18.1/centos/files/kubeadm.conf @@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' ExecStopPost=/bin/rm -f /var/run/kubelet.pid diff --git a/kubernetes/kubernetes-1.19.13/centos/files/kubeadm.conf b/kubernetes/kubernetes-1.19.13/centos/files/kubeadm.conf index 2fb25ba27..da0611444 100644 --- a/kubernetes/kubernetes-1.19.13/centos/files/kubeadm.conf +++ b/kubernetes/kubernetes-1.19.13/centos/files/kubeadm.conf @@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' ExecStopPost=/bin/rm -f /var/run/kubelet.pid diff --git a/kubernetes/kubernetes-1.20.9/centos/files/kubeadm.conf b/kubernetes/kubernetes-1.20.9/centos/files/kubeadm.conf index 2fb25ba27..da0611444 100644 --- a/kubernetes/kubernetes-1.20.9/centos/files/kubeadm.conf +++ b/kubernetes/kubernetes-1.20.9/centos/files/kubeadm.conf @@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' ExecStopPost=/bin/rm -f /var/run/kubelet.pid diff --git a/kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf b/kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf index 2fb25ba27..da0611444 100644 --- a/kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf +++ b/kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf @@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' ExecStopPost=/bin/rm -f /var/run/kubelet.pid diff --git a/kubernetes/kubernetes-1.21.8/debian/deb_folder/kubeadm.conf b/kubernetes/kubernetes-1.21.8/debian/deb_folder/kubeadm.conf index 2fb25ba27..da0611444 100644 --- a/kubernetes/kubernetes-1.21.8/debian/deb_folder/kubeadm.conf +++ b/kubernetes/kubernetes-1.21.8/debian/deb_folder/kubeadm.conf @@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' ExecStopPost=/bin/rm -f /var/run/kubelet.pid diff --git a/kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf b/kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf index 2fb25ba27..da0611444 100644 --- a/kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf +++ b/kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf @@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env EnvironmentFile=-/etc/sysconfig/kubelet ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS +ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;' ExecStopPost=/bin/rm -f /var/run/kubelet.pid diff --git a/kubernetes/kubernetes-unversioned/centos/files/sanitize_kubelet_reserved_cpus.sh b/kubernetes/kubernetes-unversioned/centos/files/sanitize_kubelet_reserved_cpus.sh new file mode 100755 index 000000000..1dcdbfdb5 --- /dev/null +++ b/kubernetes/kubernetes-unversioned/centos/files/sanitize_kubelet_reserved_cpus.sh @@ -0,0 +1,98 @@ +#! /bin/bash +# Copyright (c) 2022 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# The script will run everytime before the kubelet service is started. +# (Runs as a "ExecStartPre" action) +# +# It reads the reserved-cpus list for the kubelet from the kubelet +# environment file and sanitizes it on the basis of online CPUs. +# +# If none of the reserved cpus is online, it removes the --reserved-cpus flag +# from the environment file which allows the kubelet to choose CPUs itself +# + +ENVIRONMENT_FILE=$1 + +# Log info message to /var/log/daemon.log +function LOG { + logger -p daemon.info "$0($$): $@" +} + + +# Log error message to /var/log/daemon.log +function ERROR { + logger -s -p daemon.error "$0($$): ERROR: $@" +} + +function sanitize_reserved_cpus { + kubelet_extra_args=$(cat ${ENVIRONMENT_FILE} 2>/dev/null) + RC=$? + if [ ${RC} != "0" ]; then + ERROR "Error reading kubelet extra arguments. Error code: [${RC}]" + exit ${RC} + fi + + # Get reserved-cpus comma-separated-values string from environment file and strip double quotes + # format of kubelet_extra_args is: + # "KUBELET_EXTRA_ARGS=--cni-bin-dir=/usr/libexec/cni --node-ip=abcd:204::2 + # --system-reserved=memory=9000Mi --reserved-cpus="0-29" --pod-max-pids 10000" + if [[ ${kubelet_extra_args} =~ --reserved-cpus=\"([0-9,-]+)\" ]]; then + reserved_cpus=${BASH_REMATCH[1]} + else + reserved_cpus="" + fi + if test -z "${reserved_cpus}"; then + LOG "No reserved-cpu list found for kubelet. Nothing to do." + exit 0 + fi + LOG "Current reserved-cpus for the kubelet service: ${reserved_cpus}" + + cpus_online=$(cat /sys/devices/system/cpu/online) + RC=$? + if [ ${RC} != "0" ]; then + ERROR "Error reading online CPU list. Error code: [${RC}]" + exit ${RC} + fi + LOG "Online CPUs: ${cpus_online}" + + # Possible formats for reserved_cpus could be + # 0,2,4,6 + # 0-23,36-45 + # 0-4,6,9,13,23-34 + expanded_reserved_cpus=$(expand_sequence ${reserved_cpus}) + reserved_cpus_array=(${expanded_reserved_cpus//,/ }) + + sanitized_reserved_cpus="" + for element in "${reserved_cpus_array[@]}"; do + in_list ${element} ${cpus_online} + if [[ "$?" == "0" ]] ; then + sanitized_reserved_cpus+=",${element}" + fi + done + # Remove the extra leading ',' + sanitized_reserved_cpus=${sanitized_reserved_cpus#","} + LOG "Sanitized reserved-cpus list for the kubelet: ${sanitized_reserved_cpus}" + + if test -z "${sanitized_reserved_cpus}"; then + # Strip out --reserved-cpus option if no reserved-cpus are online + sed -i "s/ --reserved-cpus=\"${reserved_cpus}\"//g" ${ENVIRONMENT_FILE} + else + # Replace existing reserved-cpus with sanitized list + sed -i "s/--reserved-cpus=\"${reserved_cpus}\"/--reserved-cpus=\"${sanitized_reserved_cpus}\"/g" ${ENVIRONMENT_FILE} + fi + RC="$?" + if [ ${RC} != "0" ]; then + ERROR "Error updating reserved-cpus list for the kubelet. Error code: [${RC}]" + exit ${RC} + fi + LOG "Successfully updated reserved-cpus list for the kubelet." + +} + +source /etc/init.d/cpumap_functions.sh + +sanitize_reserved_cpus + +exit 0 diff --git a/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec b/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec index c9136ef0a..f9944f10c 100644 --- a/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec +++ b/kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec @@ -41,6 +41,8 @@ Source3: kubelet_override.yaml Source4: upgrade_k8s_config.sh +Source5: sanitize_kubelet_reserved_cpus.sh + Patch1: kubelet-service-remove-docker-dependency.patch BuildArch: noarch @@ -101,6 +103,8 @@ install -d %{buildroot}%{local_sbindir} # install execution scripts install -m 700 %{SOURCE4} %{buildroot}/%{local_sbindir}/upgrade_k8s_config.sh +install -m 700 %{SOURCE5} %{buildroot}/%{local_sbindir}/sanitize_kubelet_reserved_cpus.sh + # install service files install -v -d -m 0755 %{buildroot}%{_unitdir} install -v -m 0644 -t %{buildroot}%{_unitdir} contrib/init/systemd/kubelet.service @@ -120,6 +124,7 @@ install -v -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system.conf.d %{SOU # the following are execution scripts %{local_sbindir}/upgrade_k8s_config.sh +%{local_sbindir}/sanitize_kubelet_reserved_cpus.sh # the following are symlinks %{_bindir}/kubeadm diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install b/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install index 8e2cb1e78..717ad0279 100644 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/kubernetes-unversioned.install @@ -5,3 +5,4 @@ etc/kubernetes/kubelet.kubeconfig etc/kubernetes/proxy etc/systemd/system.conf.d/kubernetes-accounting.conf usr/lib/tmpfiles.d/kubernetes.conf +usr/local/sbin/sanitize_kubelet_reserved_cpus.sh diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/not-installed b/kubernetes/kubernetes-unversioned/debian/deb_folder/not-installed index 666a84f38..ae7c9c81b 100644 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/not-installed +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/not-installed @@ -1,6 +1,7 @@ usr/bin/kubeadm usr/bin/kubelet usr/bin/kubelet-cgroup-setup.sh +usr/local/sbin/sanitize_kubelet_reserved_cpus.sh usr/bin/kubectl etc/systemd/system/kubelet.service.d/kubeadm.conf usr/share/bash-completion/completions/kubectl diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/rules b/kubernetes/kubernetes-unversioned/debian/deb_folder/rules index 18921fccd..70b19abdc 100755 --- a/kubernetes/kubernetes-unversioned/debian/deb_folder/rules +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/rules @@ -5,6 +5,7 @@ _k8s_name := kubernetes _bindir := /usr/bin +_local_sbindir := /usr/local/sbin _curr_stage1 := /usr/local/kubernetes/current/stage1 _curr_stage2 := /usr/local/kubernetes/current/stage2 @@ -60,6 +61,10 @@ override_dh_install: install -v -d -m 0755 ${DEBIAN_DESTDIR}/etc/systemd/system.conf.d install -v -p -m 0644 -t ${DEBIAN_DESTDIR}/etc/systemd/system.conf.d debian/kubernetes-accounting.conf + # install scripts + install -v -m 0700 -d ${DEBIAN_DESTDIR}${_local_sbindir} + install -v -m 0700 -d ${DEBIAN_DESTDIR}${_local_sbindir}/sanitize_kubelet_reserved_cpus.sh + dh_install override_dh_usrlocal: diff --git a/kubernetes/kubernetes-unversioned/debian/deb_folder/sanitize_kubelet_reserved_cpus.sh b/kubernetes/kubernetes-unversioned/debian/deb_folder/sanitize_kubelet_reserved_cpus.sh new file mode 100755 index 000000000..1dcdbfdb5 --- /dev/null +++ b/kubernetes/kubernetes-unversioned/debian/deb_folder/sanitize_kubelet_reserved_cpus.sh @@ -0,0 +1,98 @@ +#! /bin/bash +# Copyright (c) 2022 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# The script will run everytime before the kubelet service is started. +# (Runs as a "ExecStartPre" action) +# +# It reads the reserved-cpus list for the kubelet from the kubelet +# environment file and sanitizes it on the basis of online CPUs. +# +# If none of the reserved cpus is online, it removes the --reserved-cpus flag +# from the environment file which allows the kubelet to choose CPUs itself +# + +ENVIRONMENT_FILE=$1 + +# Log info message to /var/log/daemon.log +function LOG { + logger -p daemon.info "$0($$): $@" +} + + +# Log error message to /var/log/daemon.log +function ERROR { + logger -s -p daemon.error "$0($$): ERROR: $@" +} + +function sanitize_reserved_cpus { + kubelet_extra_args=$(cat ${ENVIRONMENT_FILE} 2>/dev/null) + RC=$? + if [ ${RC} != "0" ]; then + ERROR "Error reading kubelet extra arguments. Error code: [${RC}]" + exit ${RC} + fi + + # Get reserved-cpus comma-separated-values string from environment file and strip double quotes + # format of kubelet_extra_args is: + # "KUBELET_EXTRA_ARGS=--cni-bin-dir=/usr/libexec/cni --node-ip=abcd:204::2 + # --system-reserved=memory=9000Mi --reserved-cpus="0-29" --pod-max-pids 10000" + if [[ ${kubelet_extra_args} =~ --reserved-cpus=\"([0-9,-]+)\" ]]; then + reserved_cpus=${BASH_REMATCH[1]} + else + reserved_cpus="" + fi + if test -z "${reserved_cpus}"; then + LOG "No reserved-cpu list found for kubelet. Nothing to do." + exit 0 + fi + LOG "Current reserved-cpus for the kubelet service: ${reserved_cpus}" + + cpus_online=$(cat /sys/devices/system/cpu/online) + RC=$? + if [ ${RC} != "0" ]; then + ERROR "Error reading online CPU list. Error code: [${RC}]" + exit ${RC} + fi + LOG "Online CPUs: ${cpus_online}" + + # Possible formats for reserved_cpus could be + # 0,2,4,6 + # 0-23,36-45 + # 0-4,6,9,13,23-34 + expanded_reserved_cpus=$(expand_sequence ${reserved_cpus}) + reserved_cpus_array=(${expanded_reserved_cpus//,/ }) + + sanitized_reserved_cpus="" + for element in "${reserved_cpus_array[@]}"; do + in_list ${element} ${cpus_online} + if [[ "$?" == "0" ]] ; then + sanitized_reserved_cpus+=",${element}" + fi + done + # Remove the extra leading ',' + sanitized_reserved_cpus=${sanitized_reserved_cpus#","} + LOG "Sanitized reserved-cpus list for the kubelet: ${sanitized_reserved_cpus}" + + if test -z "${sanitized_reserved_cpus}"; then + # Strip out --reserved-cpus option if no reserved-cpus are online + sed -i "s/ --reserved-cpus=\"${reserved_cpus}\"//g" ${ENVIRONMENT_FILE} + else + # Replace existing reserved-cpus with sanitized list + sed -i "s/--reserved-cpus=\"${reserved_cpus}\"/--reserved-cpus=\"${sanitized_reserved_cpus}\"/g" ${ENVIRONMENT_FILE} + fi + RC="$?" + if [ ${RC} != "0" ]; then + ERROR "Error updating reserved-cpus list for the kubelet. Error code: [${RC}]" + exit ${RC} + fi + LOG "Successfully updated reserved-cpus list for the kubelet." + +} + +source /etc/init.d/cpumap_functions.sh + +sanitize_reserved_cpus + +exit 0