Add k8s container cleanup

When executing a reboot/shutdown
k8s pods are not receiving the SIGTERM
signal which leads some of them to
unexpected behaviour such as generating
huge coredumps.

There is an upstream issue regarding this:
https://github.com/kubernetes/kubernetes/issues/107158
The problem seems to be systemd related
but this commit addresses the problem
with a workaround.

This commit introduces a new script that
will cleanup all the remaing pods and will
be run after kubelet is stopped.

The script is executed successfully when
kubelet stops and the pods are stopped
before the system shuts down.

Closes-bug: 1964111
Signed-off-by: Daniel Safta <daniel.safta@windriver.com>
Change-Id: Ia0376aa510dd0dc3983e16cd89840726c15d6c92

kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf

@@ -13,6 +13,7 @@ ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/k
 ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
 ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
 ExecStopPost=/bin/rm -f /var/run/kubelet.pid
+ExecStopPost=-/usr/local/sbin/k8s-container-cleanup
 Restart=always
 StartLimitInterval=0
 RestartSec=10

kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf

@@ -13,6 +13,7 @@ ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/k
 ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
 ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
 ExecStopPost=/bin/rm -f /var/run/kubelet.pid
+ExecStopPost=-/usr/local/sbin/k8s-container-cleanup
 Restart=always
 StartLimitInterval=0
 RestartSec=10

kubernetes/kubernetes-unversioned/centos/files/k8s-container-cleanup

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright (c) 2022 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# The script will run everytime after the kubelet service is stopped.
+#
+# It will detect any unfinished pod and will try to send them SIGTERM
+# within 5s. If it times out, SIGKILL will be sent.
+#
+
+state=$(timeout 10 systemctl is-system-running)
+
+if [ "$state" = "stopping" ]; then
+    crictl ps | cut -d ' ' -f 1  | tail -n +2 | xargs -I {} crictl stop --timeout 5 {}
+fi
+
+exit 0

kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec

@@ -43,6 +43,7 @@ Source4: upgrade_k8s_config.sh
 
 Source5: sanitize_kubelet_reserved_cpus.sh
 
+Source6: k8s-container-cleanup
 Patch1: kubelet-service-remove-docker-dependency.patch
 
 BuildArch: noarch
@@ -105,6 +106,7 @@ install -m 700 %{SOURCE4} %{buildroot}/%{local_sbindir}/upgrade_k8s_config.sh
 
 install -m 700 %{SOURCE5} %{buildroot}/%{local_sbindir}/sanitize_kubelet_reserved_cpus.sh
 
+install -m 755 %{SOURCE6} %{buildroot}%{local_sbindir}/k8s-container-cleanup
 # install service files
 install -v -d -m 0755 %{buildroot}%{_unitdir}
 install -v -m 0644 -t %{buildroot}%{_unitdir} contrib/init/systemd/kubelet.service
@@ -125,6 +127,7 @@ install -v -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system.conf.d %{SOU
 # the following are execution scripts
 %{local_sbindir}/upgrade_k8s_config.sh
 %{local_sbindir}/sanitize_kubelet_reserved_cpus.sh
+%{local_sbindir}/k8s-container-cleanup
 
 # the following are symlinks
 %{_bindir}/kubeadm