From 54868df244f625c4f360bb20ef4a2b0bea2aa3ef Mon Sep 17 00:00:00 2001
From: Zhixiong Chi <zhixiong.chi@windriver.com>
Date: Thu, 9 Mar 2023 22:35:25 -0800
Subject: [PATCH] Reorder ceph shutdown to after containers

Problem:
On node shutdown, ceph is getting shut down while it is still in use by
the pods/containers. This leads to hangs which eventually leads to the
hostwd service timing out and triggering a reboot.

Solution:
The old dependencies are not suitable for the current version of ceph
because we are now using the containerd docker runtime instead of
docker service. Meanwhile the ceph init script uses systemd-run to
launch the systemd scopes for ceph components(ceph-mon|osd|mds).
The script generates transient systemd scope files with basic
configuration.

This update patches the ceph init script to generate systemd overrides
config files for the ceph components that provide improved ordering
during shutdown. This ordering ensures kubelet and containerd services
are shut down first, then the ceph scopes and service management (SM).
As a result the timeout of hostwd service isn't triggered and the
shutdown now works properly.

TestPlan:
PASS: build-pkgs
PASS: build-image
PASS: Jenkins installation
PASS: kubectl create -f ceph-fuse.yaml
PASS: After checking the pod is running with 'kubectl get pods',
      execute the command "sudo shutdown -hP now"
PASS: The shutdown works well without os reboot.

The yaml file is as follows:
$cat ceph-fuse.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: rwx-test-claim
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 10Gi
  storageClassName: cephfs
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: wrx-centos
  namespace: default
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  selector:
    matchLabels:
      run: centos
  template:
    metadata:
      labels:
        run: centos
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: run
                operator: In
                values:
                - centos
            topologyKey: kubernetes.io/hostname
      containers:
      - name: centos
        image: centos/tools
        imagePullPolicy: IfNotPresent
        command: [ "/bin/bash", "-c", "--" ]
        args: [ "while true; do dd if=/dev/zero of=/mnt1/aaa bs=1K count=100 && sleep 1; done;" ]
        volumeMounts:
        - name: pvc1
          mountPath: "/mnt1"
      restartPolicy: Always
      volumes:
      - name: pvc1
        persistentVolumeClaim:
          claimName: rwx-test-claim

Closes-Bug: 2011610

Signed-off-by: Zhixiong Chi <zhixiong.chi@windriver.com>
Change-Id: I2c093c490ba177fbfc816e44dc227890270cac83
---
 .../debian/deb_folder/ceph-base.ceph.init     | 24 ++++++++++++++-----
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/ceph/ceph/debian/deb_folder/ceph-base.ceph.init b/ceph/ceph/debian/deb_folder/ceph-base.ceph.init
index d29d2849a..cad117111 100755
--- a/ceph/ceph/debian/deb_folder/ceph-base.ceph.init
+++ b/ceph/ceph/debian/deb_folder/ceph-base.ceph.init
@@ -990,19 +990,31 @@ for name in $what; do
 		shopt -s nullglob
 
 		OSD_SERVICES=$(for svc in /run/systemd/system/ceph-osd*.service; do basename $svc; done | xargs echo)
-	    for d in /run/systemd/system/ceph-osd*.d; do
-	        cat <<EOF > $d/starlingx-overrides.conf
+	    for d in /run/systemd/transient/ceph-osd*.scope; do
+	        do_cmd "mkdir -p $d.d"
+	        cat <<EOF > $d.d/starlingx-overrides.conf
 [Unit]
-Before=docker.service
+Before=containerd.service
 After=sm-shutdown.service
 
 EOF
 	    done
 
-	    for d in /run/systemd/system/ceph-mon*.d; do
-	        cat <<EOF > $d/starlingx-overrides.conf
+	    for d in /run/systemd/transient/ceph-mds*.scope; do
+	        do_cmd "mkdir -p $d.d"
+	        cat <<EOF > $d.d/starlingx-overrides.conf
 [Unit]
-Before=docker.service
+Before=containerd.service
+After=sm-shutdown.service
+
+EOF
+	    done
+
+	    for d in /run/systemd/transient/ceph-mon*.scope; do
+	        do_cmd "mkdir -p $d.d"
+	        cat <<EOF > $d.d/starlingx-overrides.conf
+[Unit]
+Before=containerd.service
 After=sm-shutdown.service ${OSD_SERVICES}
 
 EOF