Merge "sanitize reserved cpus list before kubelet starts"
This commit is contained in:
Zuul
Gerrit Code Review
commit
05478b39b7
@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
|
||||
EnvironmentFile=-/etc/sysconfig/kubelet
|
||||
ExecStart=
|
||||
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
|
||||
ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet
|
||||
ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
|
||||
ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
|
||||
ExecStopPost=/bin/rm -f /var/run/kubelet.pid
|
||||
|
||||
@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
|
||||
EnvironmentFile=-/etc/sysconfig/kubelet
|
||||
ExecStart=
|
||||
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
|
||||
ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet
|
||||
ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
|
||||
ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
|
||||
ExecStopPost=/bin/rm -f /var/run/kubelet.pid
|
||||
|
||||
@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
|
||||
EnvironmentFile=-/etc/sysconfig/kubelet
|
||||
ExecStart=
|
||||
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
|
||||
ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet
|
||||
ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
|
||||
ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
|
||||
ExecStopPost=/bin/rm -f /var/run/kubelet.pid
|
||||
|
||||
@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
|
||||
EnvironmentFile=-/etc/sysconfig/kubelet
|
||||
ExecStart=
|
||||
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
|
||||
ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet
|
||||
ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
|
||||
ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
|
||||
ExecStopPost=/bin/rm -f /var/run/kubelet.pid
|
||||
|
||||
@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
|
||||
EnvironmentFile=-/etc/sysconfig/kubelet
|
||||
ExecStart=
|
||||
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
|
||||
ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet
|
||||
ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
|
||||
ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
|
||||
ExecStopPost=/bin/rm -f /var/run/kubelet.pid
|
||||
|
||||
@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
|
||||
EnvironmentFile=-/etc/sysconfig/kubelet
|
||||
ExecStart=
|
||||
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
|
||||
ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/kubelet
|
||||
ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
|
||||
ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
|
||||
ExecStopPost=/bin/rm -f /var/run/kubelet.pid
|
||||
|
||||
@ -0,0 +1,98 @@
|
||||
#! /bin/bash
|
||||
# Copyright (c) 2022 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The script will run everytime before the kubelet service is started.
|
||||
# (Runs as a "ExecStartPre" action)
|
||||
#
|
||||
# It reads the reserved-cpus list for the kubelet from the kubelet
|
||||
# environment file and sanitizes it on the basis of online CPUs.
|
||||
#
|
||||
# If none of the reserved cpus is online, it removes the --reserved-cpus flag
|
||||
# from the environment file which allows the kubelet to choose CPUs itself
|
||||
#
|
||||
|
||||
ENVIRONMENT_FILE=$1
|
||||
|
||||
# Log info message to /var/log/daemon.log
|
||||
function LOG {
|
||||
logger -p daemon.info "$0($$): $@"
|
||||
}
|
||||
|
||||
|
||||
# Log error message to /var/log/daemon.log
|
||||
function ERROR {
|
||||
logger -s -p daemon.error "$0($$): ERROR: $@"
|
||||
}
|
||||
|
||||
function sanitize_reserved_cpus {
|
||||
kubelet_extra_args=$(cat ${ENVIRONMENT_FILE} 2>/dev/null)
|
||||
RC=$?
|
||||
if [ ${RC} != "0" ]; then
|
||||
ERROR "Error reading kubelet extra arguments. Error code: [${RC}]"
|
||||
exit ${RC}
|
||||
fi
|
||||
|
||||
# Get reserved-cpus comma-separated-values string from environment file and strip double quotes
|
||||
# format of kubelet_extra_args is:
|
||||
# "KUBELET_EXTRA_ARGS=--cni-bin-dir=/usr/libexec/cni --node-ip=abcd:204::2
|
||||
# --system-reserved=memory=9000Mi --reserved-cpus="0-29" --pod-max-pids 10000"
|
||||
if [[ ${kubelet_extra_args} =~ --reserved-cpus=\"([0-9,-]+)\" ]]; then
|
||||
reserved_cpus=${BASH_REMATCH[1]}
|
||||
else
|
||||
reserved_cpus=""
|
||||
fi
|
||||
if test -z "${reserved_cpus}"; then
|
||||
LOG "No reserved-cpu list found for kubelet. Nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
LOG "Current reserved-cpus for the kubelet service: ${reserved_cpus}"
|
||||
|
||||
cpus_online=$(cat /sys/devices/system/cpu/online)
|
||||
RC=$?
|
||||
if [ ${RC} != "0" ]; then
|
||||
ERROR "Error reading online CPU list. Error code: [${RC}]"
|
||||
exit ${RC}
|
||||
fi
|
||||
LOG "Online CPUs: ${cpus_online}"
|
||||
|
||||
# Possible formats for reserved_cpus could be
|
||||
# 0,2,4,6
|
||||
# 0-23,36-45
|
||||
# 0-4,6,9,13,23-34
|
||||
expanded_reserved_cpus=$(expand_sequence ${reserved_cpus})
|
||||
reserved_cpus_array=(${expanded_reserved_cpus//,/ })
|
||||
|
||||
sanitized_reserved_cpus=""
|
||||
for element in "${reserved_cpus_array[@]}"; do
|
||||
in_list ${element} ${cpus_online}
|
||||
if [[ "$?" == "0" ]] ; then
|
||||
sanitized_reserved_cpus+=",${element}"
|
||||
fi
|
||||
done
|
||||
# Remove the extra leading ','
|
||||
sanitized_reserved_cpus=${sanitized_reserved_cpus#","}
|
||||
LOG "Sanitized reserved-cpus list for the kubelet: ${sanitized_reserved_cpus}"
|
||||
|
||||
if test -z "${sanitized_reserved_cpus}"; then
|
||||
# Strip out --reserved-cpus option if no reserved-cpus are online
|
||||
sed -i "s/ --reserved-cpus=\"${reserved_cpus}\"//g" ${ENVIRONMENT_FILE}
|
||||
else
|
||||
# Replace existing reserved-cpus with sanitized list
|
||||
sed -i "s/--reserved-cpus=\"${reserved_cpus}\"/--reserved-cpus=\"${sanitized_reserved_cpus}\"/g" ${ENVIRONMENT_FILE}
|
||||
fi
|
||||
RC="$?"
|
||||
if [ ${RC} != "0" ]; then
|
||||
ERROR "Error updating reserved-cpus list for the kubelet. Error code: [${RC}]"
|
||||
exit ${RC}
|
||||
fi
|
||||
LOG "Successfully updated reserved-cpus list for the kubelet."
|
||||
|
||||
}
|
||||
|
||||
source /etc/init.d/cpumap_functions.sh
|
||||
|
||||
sanitize_reserved_cpus
|
||||
|
||||
exit 0
|
||||
@ -41,6 +41,8 @@ Source3: kubelet_override.yaml
|
||||
|
||||
Source4: upgrade_k8s_config.sh
|
||||
|
||||
Source5: sanitize_kubelet_reserved_cpus.sh
|
||||
|
||||
Patch1: kubelet-service-remove-docker-dependency.patch
|
||||
|
||||
BuildArch: noarch
|
||||
@ -101,6 +103,8 @@ install -d %{buildroot}%{local_sbindir}
|
||||
# install execution scripts
|
||||
install -m 700 %{SOURCE4} %{buildroot}/%{local_sbindir}/upgrade_k8s_config.sh
|
||||
|
||||
install -m 700 %{SOURCE5} %{buildroot}/%{local_sbindir}/sanitize_kubelet_reserved_cpus.sh
|
||||
|
||||
# install service files
|
||||
install -v -d -m 0755 %{buildroot}%{_unitdir}
|
||||
install -v -m 0644 -t %{buildroot}%{_unitdir} contrib/init/systemd/kubelet.service
|
||||
@ -120,6 +124,7 @@ install -v -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system.conf.d %{SOU
|
||||
|
||||
# the following are execution scripts
|
||||
%{local_sbindir}/upgrade_k8s_config.sh
|
||||
%{local_sbindir}/sanitize_kubelet_reserved_cpus.sh
|
||||
|
||||
# the following are symlinks
|
||||
%{_bindir}/kubeadm
|
||||
|
||||
@ -5,3 +5,4 @@ etc/kubernetes/kubelet.kubeconfig
|
||||
etc/kubernetes/proxy
|
||||
etc/systemd/system.conf.d/kubernetes-accounting.conf
|
||||
usr/lib/tmpfiles.d/kubernetes.conf
|
||||
usr/local/sbin/sanitize_kubelet_reserved_cpus.sh
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
usr/bin/kubeadm
|
||||
usr/bin/kubelet
|
||||
usr/bin/kubelet-cgroup-setup.sh
|
||||
usr/local/sbin/sanitize_kubelet_reserved_cpus.sh
|
||||
usr/bin/kubectl
|
||||
etc/systemd/system/kubelet.service.d/kubeadm.conf
|
||||
usr/share/bash-completion/completions/kubectl
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
|
||||
_k8s_name := kubernetes
|
||||
_bindir := /usr/bin
|
||||
_local_sbindir := /usr/local/sbin
|
||||
_curr_stage1 := /usr/local/kubernetes/current/stage1
|
||||
_curr_stage2 := /usr/local/kubernetes/current/stage2
|
||||
|
||||
@ -60,6 +61,10 @@ override_dh_install:
|
||||
install -v -d -m 0755 ${DEBIAN_DESTDIR}/etc/systemd/system.conf.d
|
||||
install -v -p -m 0644 -t ${DEBIAN_DESTDIR}/etc/systemd/system.conf.d debian/kubernetes-accounting.conf
|
||||
|
||||
# install scripts
|
||||
install -v -m 0700 -d ${DEBIAN_DESTDIR}${_local_sbindir}
|
||||
install -v -m 0700 -d ${DEBIAN_DESTDIR}${_local_sbindir}/sanitize_kubelet_reserved_cpus.sh
|
||||
|
||||
dh_install
|
||||
|
||||
override_dh_usrlocal:
|
||||
|
||||
@ -0,0 +1,98 @@
|
||||
#! /bin/bash
|
||||
# Copyright (c) 2022 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The script will run everytime before the kubelet service is started.
|
||||
# (Runs as a "ExecStartPre" action)
|
||||
#
|
||||
# It reads the reserved-cpus list for the kubelet from the kubelet
|
||||
# environment file and sanitizes it on the basis of online CPUs.
|
||||
#
|
||||
# If none of the reserved cpus is online, it removes the --reserved-cpus flag
|
||||
# from the environment file which allows the kubelet to choose CPUs itself
|
||||
#
|
||||
|
||||
ENVIRONMENT_FILE=$1
|
||||
|
||||
# Log info message to /var/log/daemon.log
|
||||
function LOG {
|
||||
logger -p daemon.info "$0($$): $@"
|
||||
}
|
||||
|
||||
|
||||
# Log error message to /var/log/daemon.log
|
||||
function ERROR {
|
||||
logger -s -p daemon.error "$0($$): ERROR: $@"
|
||||
}
|
||||
|
||||
function sanitize_reserved_cpus {
|
||||
kubelet_extra_args=$(cat ${ENVIRONMENT_FILE} 2>/dev/null)
|
||||
RC=$?
|
||||
if [ ${RC} != "0" ]; then
|
||||
ERROR "Error reading kubelet extra arguments. Error code: [${RC}]"
|
||||
exit ${RC}
|
||||
fi
|
||||
|
||||
# Get reserved-cpus comma-separated-values string from environment file and strip double quotes
|
||||
# format of kubelet_extra_args is:
|
||||
# "KUBELET_EXTRA_ARGS=--cni-bin-dir=/usr/libexec/cni --node-ip=abcd:204::2
|
||||
# --system-reserved=memory=9000Mi --reserved-cpus="0-29" --pod-max-pids 10000"
|
||||
if [[ ${kubelet_extra_args} =~ --reserved-cpus=\"([0-9,-]+)\" ]]; then
|
||||
reserved_cpus=${BASH_REMATCH[1]}
|
||||
else
|
||||
reserved_cpus=""
|
||||
fi
|
||||
if test -z "${reserved_cpus}"; then
|
||||
LOG "No reserved-cpu list found for kubelet. Nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
LOG "Current reserved-cpus for the kubelet service: ${reserved_cpus}"
|
||||
|
||||
cpus_online=$(cat /sys/devices/system/cpu/online)
|
||||
RC=$?
|
||||
if [ ${RC} != "0" ]; then
|
||||
ERROR "Error reading online CPU list. Error code: [${RC}]"
|
||||
exit ${RC}
|
||||
fi
|
||||
LOG "Online CPUs: ${cpus_online}"
|
||||
|
||||
# Possible formats for reserved_cpus could be
|
||||
# 0,2,4,6
|
||||
# 0-23,36-45
|
||||
# 0-4,6,9,13,23-34
|
||||
expanded_reserved_cpus=$(expand_sequence ${reserved_cpus})
|
||||
reserved_cpus_array=(${expanded_reserved_cpus//,/ })
|
||||
|
||||
sanitized_reserved_cpus=""
|
||||
for element in "${reserved_cpus_array[@]}"; do
|
||||
in_list ${element} ${cpus_online}
|
||||
if [[ "$?" == "0" ]] ; then
|
||||
sanitized_reserved_cpus+=",${element}"
|
||||
fi
|
||||
done
|
||||
# Remove the extra leading ','
|
||||
sanitized_reserved_cpus=${sanitized_reserved_cpus#","}
|
||||
LOG "Sanitized reserved-cpus list for the kubelet: ${sanitized_reserved_cpus}"
|
||||
|
||||
if test -z "${sanitized_reserved_cpus}"; then
|
||||
# Strip out --reserved-cpus option if no reserved-cpus are online
|
||||
sed -i "s/ --reserved-cpus=\"${reserved_cpus}\"//g" ${ENVIRONMENT_FILE}
|
||||
else
|
||||
# Replace existing reserved-cpus with sanitized list
|
||||
sed -i "s/--reserved-cpus=\"${reserved_cpus}\"/--reserved-cpus=\"${sanitized_reserved_cpus}\"/g" ${ENVIRONMENT_FILE}
|
||||
fi
|
||||
RC="$?"
|
||||
if [ ${RC} != "0" ]; then
|
||||
ERROR "Error updating reserved-cpus list for the kubelet. Error code: [${RC}]"
|
||||
exit ${RC}
|
||||
fi
|
||||
LOG "Successfully updated reserved-cpus list for the kubelet."
|
||||
|
||||
}
|
||||
|
||||
source /etc/init.d/cpumap_functions.sh
|
||||
|
||||
sanitize_reserved_cpus
|
||||
|
||||
exit 0
|
||||
Loading…
x
Reference in New Issue
Block a user