Move k8s container cleanup to containerd service

This introduces k8s-container-cleanup script that will be called
when containerd.service is stopped. The script detects whether systemd
state is 'stopping' due to shutdown/reboot, then stops all running
containers before the service shuts down.

During shutdown/reboot, some containers are not receiving the
SIGTERM signal. This leads to unexpected behaviour such as
generating huge coredumps.

There is an upstream issue regarding this:
https://github.com/kubernetes/kubernetes/issues/107158
The problem seems to be systemd related but this commit
addresses the problem with a workaround.

This reverts commit f3c18b0f79.
The k8s-container-cleanup script is moved from kubelet.service
to containerd.service. The ExecStopPost that calls this script
is removed, and replaced with ExecStop in containerd.service
to call the script (in config-files repo).

The k8s-container-cleanup script requires containerd is running
in order to use crictl utility. The shutdown of kubelet and
containerd have unpredictable timing, so the cleanup must be done
in containerd.

Test Plan: On AIO-SX
PASS: Verify k8s-container-cleanup logs to daemon.log during 'stopping.
PASS: Manual change containerd/kubelet shutdown timing and verify.
k8s-container-cleanup running to completion before containerd stopped.
PASS: Reboot and verify k8s-container-cleanup running to completion.
PASS: Lock/unlock and verify k8s-container-cleanup running to completion.
PASS: Manually run spellintian tool against k8s-container-cleanup.sh.
PASS: Manually run shellcheck tool against k8s-container-cleanup.sh.
PASS: Zuul tox bashate tool against k8s-container-cleanup.sh.

Partial-Bug: 1964111
Change-Id: Ic8a9e257f861ae218a8520205eced3eaa580dd20
Signed-off-by: Jim Gauld <james.gauld@windriver.com>

kubernetes/containerd/centos/containerd.spec

@@ -12,6 +12,7 @@ Source0: containerd-v%{version}.tar.gz
 Source1: runc-1.0.2.tar.gz
 Source2: crictl-v1.21.0-linux-amd64.tar.gz
 Source3: crictl.yaml
+Source4: k8s-container-cleanup.sh
 Patch1: 0001-customize-containerd-for-StarlingX.patch
 Patch2: 0002-CRI-Reduce-clutter-of-log-entries-during-process-exe.patch
 URL: https://www.starlingx.io
@@ -43,6 +44,7 @@ Provides: containerd.io
 
 %global _missing_build_ids_terminate_build 0
 
+%define local_sbindir /usr/local/sbin
 %define CONTAINERD_DIR ${HOME}/go/src/github.com/containerd/containerd
 %define RUNC_DIR ${HOME}/go/src/github.com/opencontainers/runc
 
@@ -94,6 +96,8 @@ install -d %{buildroot}%{_sysconfdir}
 install -m 644 %{_sourcedir}/crictl.yaml %{buildroot}%{_sysconfdir}/crictl.yaml
 install -d %{buildroot}%{_unitdir}
 install -p -m 644 %{CONTAINERD_DIR}/containerd.service %{buildroot}%{_unitdir}/containerd.service
+install -d %{buildroot}%{local_sbindir}
+install -m 755 %{SOURCE4} %{buildroot}%{local_sbindir}/k8s-container-cleanup
 
 # list files owned by the package here
 %files
@@ -108,3 +112,4 @@ install -p -m 644 %{CONTAINERD_DIR}/containerd.service %{buildroot}%{_unitdir}/c
 %{_bindir}/crictl
 %{_sysconfdir}/crictl.yaml
 %{_unitdir}/containerd.service
+%{local_sbindir}/k8s-container-cleanup

kubernetes/containerd/centos/files/k8s-container-cleanup.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Copyright (c) 2022 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# The script will run during containerd.service ExecStop.
+# This script detects whether systemd state is 'stopping' due to
+# shutdown/reboot, then will stop all running containers before the
+# service shuts down.
+#
+# All running containers are stopped one container at a time.
+# The internal implementation of 'crictl stop --timeout <n>'
+# sends a SIGTERM to the container, and will use SIGKILL only
+# if the timeout is reached.
+#
+
+NAME=$(basename "${0}")
+
+# Log info message to /var/log/daemon.log
+function LOG {
+    logger -p daemon.info -t "${NAME}($$): " "${@}"
+}
+
+# Log error message to /var/log/daemon.log
+function ERROR {
+    logger -p daemon.error -t "${NAME}($$): " "${@}"
+}
+
+state=$(timeout 10 systemctl is-system-running)
+RC=$?
+LOG "System state is: ${state}, RC = ${RC}."
+case $RC in
+    124)
+        # systemctl hung.
+        ERROR "systemctl timed out. System state unknown."
+        ;;
+
+    [01])
+        # 0 - running; 1 - initializing, starting, degraded, maintenance, stopping
+        if [ "$state" = "stopping" ]; then
+            LOG "Stopping all containers."
+            # Use crictl to gracefully stop each container. If specified timeout is
+            # reached, it forcibly kills the container. There is no need to check
+            # return code since there is nothing more we can do, and crictl already
+            # logs to daemon.log.
+            crictl ps -q | xargs -r -I {} crictl stop --timeout 5 {}
+            LOG "Stopping all containers completed."
+            exit 0
+        fi
+        ;;
+esac
+
+exit 0

kubernetes/kubernetes-1.21.8/centos/files/kubeadm.conf

@@ -13,7 +13,6 @@ ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/k
 ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
 ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
 ExecStopPost=/bin/rm -f /var/run/kubelet.pid
-ExecStopPost=-/usr/local/sbin/k8s-container-cleanup
 Restart=always
 StartLimitInterval=0
 RestartSec=10

kubernetes/kubernetes-1.22.5/centos/files/kubeadm.conf

@@ -13,7 +13,6 @@ ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/k
 ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
 ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
 ExecStopPost=/bin/rm -f /var/run/kubelet.pid
-ExecStopPost=-/usr/local/sbin/k8s-container-cleanup
 Restart=always
 StartLimitInterval=0
 RestartSec=10

kubernetes/kubernetes-1.23.1/centos/files/kubeadm.conf

@@ -13,7 +13,6 @@ ExecStartPre=-/usr/local/sbin/sanitize_kubelet_reserved_cpus.sh /etc/sysconfig/k
 ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
 ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
 ExecStopPost=/bin/rm -f /var/run/kubelet.pid
-ExecStopPost=-/usr/local/sbin/k8s-container-cleanup
 Restart=always
 StartLimitInterval=0
 RestartSec=10

kubernetes/kubernetes-unversioned/centos/files/k8s-container-cleanup

@@ -1,18 +0,0 @@
-#!/bin/bash
-# Copyright (c) 2022 Wind River Systems, Inc.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-# The script will run everytime after the kubelet service is stopped.
-#
-# It will detect any unfinished pod and will try to send them SIGTERM
-# within 5s. If it times out, SIGKILL will be sent.
-#
-
-state=$(timeout 10 systemctl is-system-running)
-
-if [ "$state" = "stopping" ]; then
-    crictl ps | cut -d ' ' -f 1  | tail -n +2 | xargs -I {} crictl stop --timeout 5 {}
-fi
-
-exit 0

kubernetes/kubernetes-unversioned/centos/kubernetes-unversioned.spec

@@ -43,7 +43,6 @@ Source4: upgrade_k8s_config.sh
 
 Source5: sanitize_kubelet_reserved_cpus.sh
 
-Source6: k8s-container-cleanup
 Patch1: kubelet-service-remove-docker-dependency.patch
 
 BuildArch: noarch
@@ -106,7 +105,6 @@ install -m 700 %{SOURCE4} %{buildroot}/%{local_sbindir}/upgrade_k8s_config.sh
 
 install -m 700 %{SOURCE5} %{buildroot}/%{local_sbindir}/sanitize_kubelet_reserved_cpus.sh
 
-install -m 755 %{SOURCE6} %{buildroot}%{local_sbindir}/k8s-container-cleanup
 # install service files
 install -v -d -m 0755 %{buildroot}%{_unitdir}
 install -v -m 0644 -t %{buildroot}%{_unitdir} contrib/init/systemd/kubelet.service
@@ -127,7 +125,6 @@ install -v -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system.conf.d %{SOU
 # the following are execution scripts
 %{local_sbindir}/upgrade_k8s_config.sh
 %{local_sbindir}/sanitize_kubelet_reserved_cpus.sh
-%{local_sbindir}/k8s-container-cleanup
 
 # the following are symlinks
 %{_bindir}/kubeadm