From 6bd45c96dd05e819d70140f6fe4d27e073b3988a Mon Sep 17 00:00:00 2001
From: Jim Gauld <james.gauld@windriver.com>
Date: Fri, 10 May 2019 12:57:37 -0400
Subject: [PATCH] Create k8s-infra cgroup path before kubelet launch

This adds a kubelet ExecStartPre script to ensure cgroup is setup
prior to kubelet launch. This creates k8s-infra cgroup for a minimal
set of resource controllers, and configures cpuset attributes to span
all online cpus and nodes. This will do nothing if the k8s-infra cgroup
already exists (i.e., assume already configured).

NOTE: The creation of directories under /sys/fs/cgroup is volatile, and
does not persist reboots. The cpuset.mems and cpuset.cpus is later
updated by puppet kubernetes.pp manifest.

Tests performed:
Standard system: system install, lock/unlock controller & computes,
forced reboot: active/standby controller, computes.

Change-Id: I6a7aad5c40fe8225e9e16c8d8b40a0cffd76715d
Closes-Bug: 1828270
Signed-off-by: Jim Gauld <james.gauld@windriver.com>
---
 .../kubernetes/centos/files/kubeadm.conf      |   1 +
 .../centos/files/kubelet-cgroup-setup.sh      | 114 ++++++++++++++++++
 kubernetes/kubernetes/centos/kubernetes.spec  |   5 +
 3 files changed, 120 insertions(+)
 create mode 100644 kubernetes/kubernetes/centos/files/kubelet-cgroup-setup.sh

diff --git a/kubernetes/kubernetes/centos/files/kubeadm.conf b/kubernetes/kubernetes/centos/files/kubeadm.conf
index 7bba1dd45..2fb25ba27 100644
--- a/kubernetes/kubernetes/centos/files/kubeadm.conf
+++ b/kubernetes/kubernetes/centos/files/kubeadm.conf
@@ -9,6 +9,7 @@ EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
 EnvironmentFile=-/etc/sysconfig/kubelet
 ExecStart=
 ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
+ExecStartPre=-/usr/bin/kubelet-cgroup-setup.sh
 ExecStartPost=/bin/bash -c 'echo $MAINPID > /var/run/kubelet.pid;'
 ExecStopPost=/bin/rm -f /var/run/kubelet.pid
 Restart=always
diff --git a/kubernetes/kubernetes/centos/files/kubelet-cgroup-setup.sh b/kubernetes/kubernetes/centos/files/kubelet-cgroup-setup.sh
new file mode 100644
index 000000000..202348a89
--- /dev/null
+++ b/kubernetes/kubernetes/centos/files/kubelet-cgroup-setup.sh
@@ -0,0 +1,114 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This script does minimal cgroup setup for kubelet. This creates k8s-infra
+# cgroup for a minimal set of resource controllers, and configures cpuset
+# attributes to span all online cpus and nodes. This will do nothing if
+# the k8s-infra cgroup already exists (i.e., assume already configured).
+# NOTE: The creation of directories under /sys/fs/cgroup is volatile, and
+# does not persist reboots. The cpuset.mems and cpuset.cpus is later updated
+# by puppet kubernetes.pp manifest.
+#
+
+# Define minimal path
+PATH=/bin:/usr/bin:/usr/local/bin
+
+# Log info message to /var/log/daemon.log
+function LOG {
+    logger -p daemon.info "$0($$): $@"
+}
+
+# Log error message to /var/log/daemon.log
+function ERROR {
+    logger -s -p daemon.error "$0($$): ERROR: $@"
+}
+
+# Create minimal cgroup directories and configure cpuset attributes
+function create_cgroup {
+    local cg_name=$1
+    local cg_nodeset=$2
+    local cg_cpuset=$3
+
+    local CGROUP=/sys/fs/cgroup
+    local CONTROLLERS=("cpuset" "memory" "cpu,cpuacct" "systemd")
+    local cnt=''
+    local CGDIR=''
+    local RC=0
+
+    # Create the cgroup for required controllers
+    for cnt in ${CONTROLLERS[@]}; do
+        CGDIR=${CGROUP}/${cnt}/${cg_name}
+        if [ -d ${CGDIR} ]; then
+            LOG "Nothing to do, already configured: ${CGDIR}."
+            exit ${RC}
+        fi
+        LOG "Creating: ${CGDIR}"
+        mkdir -p ${CGDIR}
+        RC=$?
+        if [ ${RC} -ne 0 ]; then
+            ERROR "Creating: ${CGDIR}, rc=${RC}"
+            exit ${RC}
+        fi
+    done
+
+    # Customize cpuset attributes
+    LOG "Configuring cgroup: ${cg_name}, nodeset: ${cg_nodeset}, cpuset: ${cg_cpuset}"
+    CGDIR=${CGROUP}/cpuset/${cg_name}
+    local CGMEMS=${CGDIR}/cpuset.mems
+    local CGCPUS=${CGDIR}/cpuset.cpus
+    local CGTASKS=${CGDIR}/tasks
+
+    # Assign cgroup memory nodeset
+    LOG "Assign nodeset ${cg_nodeset} to ${CGMEMS}"
+    /bin/echo ${cg_nodeset} > ${CGMEMS}
+    RC=$?
+    if [ ${RC} -ne 0 ]; then
+        ERROR "Unable to write to: ${CGMEMS}, rc=${RC}"
+        exit ${RC}
+    fi
+
+    # Assign cgroup cpus
+    LOG "Assign cpuset ${cg_cpuset} to ${CGCPUS}"
+    /bin/echo ${cg_cpuset} > ${CGCPUS}
+    RC=$?
+    if [ ${RC} -ne 0 ]; then
+        ERROR "Assigning: ${cg_cpuset} to ${CGCPUS}, rc=${RC}"
+        exit ${RC}
+    fi
+
+    # Set file ownership
+    chown root:root ${CGMEMS} ${CGCPUS} ${CGTASKS}
+    RC=$?
+    if [ ${RC} -ne 0 ]; then
+        ERROR "Setting owner for: ${CGMEMS}, ${CGCPUS}, ${CGTASKS}, rc=${RC}"
+        exit ${RC}
+    fi
+
+    # Set file mode permissions
+    chmod 644 ${CGMEMS} ${CGCPUS} ${CGTASKS}
+    RC=$?
+    if [ ${RC} -ne 0 ]; then
+        ERROR "Setting mode for: ${CGMEMS}, ${CGCPUS}, ${CGTASKS}, rc=${RC}"
+        exit ${RC}
+    fi
+
+    return ${RC}
+}
+
+if [ $UID -ne 0 ]; then
+    ERROR "Require sudo/root."
+    exit 1
+fi
+
+# Configure default kubepods cpuset to span all online cpus and nodes.
+ONLINE_NODESET=$(/bin/cat /sys/devices/system/node/online)
+ONLINE_CPUSET=$(/bin/cat /sys/devices/system/cpu/online)
+
+# Configure kubelet cgroup to match cgroupRoot.
+create_cgroup 'k8s-infra' ${ONLINE_NODESET} ${ONLINE_CPUSET}
+
+exit $?
+
diff --git a/kubernetes/kubernetes/centos/kubernetes.spec b/kubernetes/kubernetes/centos/kubernetes.spec
index 727099004..23cca76c7 100644
--- a/kubernetes/kubernetes/centos/kubernetes.spec
+++ b/kubernetes/kubernetes/centos/kubernetes.spec
@@ -51,6 +51,7 @@ Source0:        %{project}-v%{kube_version}.tar.gz
 Source1:        %{con_repo}-v%{con_commit}.tar.gz
 Source3:        kubernetes-accounting.conf
 Source4:        kubeadm.conf
+Source5:        kubelet-cgroup-setup.sh
 
 Source33:       genmanpages.sh
 
@@ -914,6 +915,9 @@ install -p -m 755 -t %{buildroot}%{_bindir} ${output_path}/kubeadm
 install -d -m 0755 %{buildroot}/%{_sysconfdir}/systemd/system/kubelet.service.d
 install -p -m 0644 -t %{buildroot}/%{_sysconfdir}/systemd/system/kubelet.service.d %{SOURCE4}
 
+echo "+++ INSTALLING kubelet-cgroup-setup.sh"
+install -p -m 0700 -t %{buildroot}/%{_bindir} %{SOURCE5}
+
 binaries=(kube-controller-manager kube-scheduler kube-proxy kubelet kubectl)
 for bin in "${binaries[@]}"; do
   echo "+++ HARDLINKING ${bin} to hyperkube"
@@ -1038,6 +1042,7 @@ fi
 %{_mandir}/man1/kubelet.1*
 %{_mandir}/man1/kube-proxy.1*
 %{_bindir}/kubelet
+%{_bindir}/kubelet-cgroup-setup.sh
 %{_bindir}/kube-proxy
 %{_bindir}/hyperkube
 %{_unitdir}/kube-proxy.service