From 2b500465dae3f77cbffcd80fc2c65444f0bc28e6 Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Thu, 12 Dec 2024 00:52:33 +0000 Subject: [PATCH] [ceph] Fix for ceph-osd pods restart This PS updates ceph-osd pod containers making sure that osd pods are not stuck at deletion. In this PS we are taking care of another background process that has to be terminated by preStop script. Change-Id: Icebb6119225b4b88fb213932cc3bcf78d650848f --- ceph-osd/Chart.yaml | 2 +- ceph-osd/templates/bin/osd/_log-runner-stop.sh.tpl | 7 +++++++ ceph-osd/templates/bin/osd/_log-tail.sh.tpl | 4 +++- releasenotes/notes/ceph-osd.yaml | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ceph-osd/Chart.yaml b/ceph-osd/Chart.yaml index 0d52d074b..41c556e27 100644 --- a/ceph-osd/Chart.yaml +++ b/ceph-osd/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph OSD name: ceph-osd -version: 0.1.56 +version: 0.1.57 home: https://github.com/ceph/ceph ... diff --git a/ceph-osd/templates/bin/osd/_log-runner-stop.sh.tpl b/ceph-osd/templates/bin/osd/_log-runner-stop.sh.tpl index eed9dbb75..4658c9855 100644 --- a/ceph-osd/templates/bin/osd/_log-runner-stop.sh.tpl +++ b/ceph-osd/templates/bin/osd/_log-runner-stop.sh.tpl @@ -26,3 +26,10 @@ do kill -9 ${TAIL_PID}; sleep 1; done + +SLEEP_PID="$(cat /tmp/ceph-log-runner-sleep.pid)" +while kill -0 ${SLEEP_PID} >/dev/null 2>&1; +do + kill -9 ${SLEEP_PID}; + sleep 1; +done diff --git a/ceph-osd/templates/bin/osd/_log-tail.sh.tpl b/ceph-osd/templates/bin/osd/_log-tail.sh.tpl index 541aa5fbf..e903760fe 100644 --- a/ceph-osd/templates/bin/osd/_log-tail.sh.tpl +++ b/ceph-osd/templates/bin/osd/_log-tail.sh.tpl @@ -25,7 +25,7 @@ function tail_file () { while $keep_running; do tail --retry -f "${log_file}" & tail_pid=$! - echo $tail_pid > /tmp/ceph-log-runner.pid + echo $tail_pid > /tmp/ceph-log-runner-tail.pid wait $tail_pid if [ -f /tmp/ceph-log-runner.stop ]; then keep_running=false @@ -37,6 +37,8 @@ function tail_file () { function truncate_log () { while $keep_running; do sleep ${TRUNCATE_PERIOD} + sleep_pid=$! + echo $sleep_pid > /tmp/ceph-log-runner-sleep.pid if [[ -f ${log_file} ]] ; then truncate -s "${TRUNCATE_SIZE}" "${log_file}" fi diff --git a/releasenotes/notes/ceph-osd.yaml b/releasenotes/notes/ceph-osd.yaml index 165a9c522..2fc30d842 100644 --- a/releasenotes/notes/ceph-osd.yaml +++ b/releasenotes/notes/ceph-osd.yaml @@ -57,4 +57,5 @@ ceph-osd: - 0.1.54 Remove use of tini for ceph-daemon - 0.1.55 Update ceph-osd pod containers to make sure OSD pods are properly terminated at restart - 0.1.56 Add preStop lifecycle script to log-runner + - 0.1.57 Added code to kill another background process in log-runner at restart ...