Merge "Add ceph entrypoint scripts"

ceph/templates/bin/_common_functions.sh.tpl

@@ -0,0 +1,223 @@
+#!/bin/bash
+set -ex
+
+# log arguments with timestamp
+function log {
+  if [ -z "$*" ]; then
+    return 1
+  fi
+
+  TIMESTAMP=$(date '+%F %T')
+  echo "${TIMESTAMP}  $0: $*"
+  return 0
+}
+
+# ceph config file exists or die
+function check_config {
+  if [[ ! -e /etc/ceph/${CLUSTER}.conf ]]; then
+    log "ERROR- /etc/ceph/${CLUSTER}.conf must exist; get it from your existing mon"
+    exit 1
+  fi
+}
+
+# ceph admin key exists or die
+function check_admin_key {
+  if [[ ! -e $ADMIN_KEYRING ]]; then
+      log "ERROR- $ADMIN_KEYRING must exist; get it from your existing mon"
+      exit 1
+  fi
+}
+
+# Given two strings, return the length of the shared prefix
+function prefix_length {
+  local maxlen=${#1}
+  for ((i=maxlen-1;i>=0;i--)); do
+    if [[ "${1:0:i}" == "${2:0:i}" ]]; then
+      echo $i
+      return
+    fi
+  done
+}
+
+# Test if a command line tool is available
+function is_available {
+  command -v $@ &>/dev/null
+}
+
+# create the mandatory directories
+function create_mandatory_directories {
+  # Let's create the bootstrap directories
+  for keyring in $OSD_BOOTSTRAP_KEYRING $MDS_BOOTSTRAP_KEYRING $RGW_BOOTSTRAP_KEYRING; do
+    mkdir -p $(dirname $keyring)
+  done
+
+  # Let's create the ceph directories
+  for directory in mon osd mds radosgw tmp mgr; do
+    mkdir -p /var/lib/ceph/$directory
+  done
+
+  # Make the monitor directory
+  mkdir -p "$MON_DATA_DIR"
+
+  # Create socket directory
+  mkdir -p /var/run/ceph
+
+  # Creating rados directories
+  mkdir -p /var/lib/ceph/radosgw/${RGW_NAME}
+
+  # Create the MDS directory
+  mkdir -p /var/lib/ceph/mds/${CLUSTER}-${MDS_NAME}
+
+  # Create the MGR directory
+  mkdir -p /var/lib/ceph/mgr/${CLUSTER}-$MGR_NAME
+
+  # Adjust the owner of all those directories
+  chown -R ceph. /var/run/ceph/ /var/lib/ceph/*
+}
+
+
+# Calculate proper device names, given a device and partition number
+function dev_part {
+  local osd_device=${1}
+  local osd_partition=${2}
+
+  if [[ -L ${osd_device} ]]; then
+    # This device is a symlink. Work out it's actual device
+    local actual_device=$(readlink -f ${osd_device})
+    local bn=$(basename ${osd_device})
+    if [[ "${actual_device:0-1:1}" == [0-9] ]]; then
+      local desired_partition="${actual_device}p${osd_partition}"
+    else
+      local desired_partition="${actual_device}${osd_partition}"
+    fi
+    # Now search for a symlink in the directory of $osd_device
+    # that has the correct desired partition, and the longest
+    # shared prefix with the original symlink
+    local symdir=$(dirname ${osd_device})
+    local link=""
+    local pfxlen=0
+    for option in $(ls $symdir); do
+    if [[ $(readlink -f $symdir/$option) == $desired_partition ]]; then
+      local optprefixlen=$(prefix_length $option $bn)
+      if [[ $optprefixlen > $pfxlen ]]; then
+        link=$symdir/$option
+        pfxlen=$optprefixlen
+      fi
+    fi
+    done
+    if [[ $pfxlen -eq 0 ]]; then
+      >&2 log "Could not locate appropriate symlink for partition ${osd_partition} of ${osd_device}"
+      exit 1
+    fi
+    echo "$link"
+  elif [[ "${osd_device:0-1:1}" == [0-9] ]]; then
+    echo "${osd_device}p${osd_partition}"
+  else
+    echo "${osd_device}${osd_partition}"
+  fi
+}
+
+function osd_trying_to_determine_scenario {
+  if [ -z "${OSD_DEVICE}" ]; then
+    log "Bootstrapped OSD(s) found; using OSD directory"
+    source osd_directory.sh
+    osd_directory
+  elif $(parted --script ${OSD_DEVICE} print | egrep -sq '^ 1.*ceph data'); then
+    log "Bootstrapped OSD found; activating ${OSD_DEVICE}"
+    source osd_disk_activate.sh
+    osd_activate
+  else
+    log "Device detected, assuming ceph-disk scenario is desired"
+    log "Preparing and activating ${OSD_DEVICE}"
+    osd_disk
+  fi
+}
+
+function get_osd_dev {
+  for i in ${OSD_DISKS}
+   do
+    osd_id=$(echo ${i}|sed 's/\(.*\):\(.*\)/\1/')
+    osd_dev="/dev/$(echo ${i}|sed 's/\(.*\):\(.*\)/\2/')"
+    if [ ${osd_id} = ${1} ]; then
+      echo -n "${osd_dev}"
+    fi
+  done
+}
+
+function unsupported_scenario {
+  echo "ERROR: '${CEPH_DAEMON}' scenario or key/value store '${KV_TYPE}' is not supported by this distribution."
+  echo "ERROR: for the list of supported scenarios, please refer to your vendor."
+  exit 1
+}
+
+function is_integer {
+  # This function is about saying if the passed argument is an integer
+  # Supports also negative integers
+  # We use $@ here to consider everything given as parameter and not only the
+  # first one : that's mainly for splited strings like "10 10"
+  [[ $@ =~ ^-?[0-9]+$ ]]
+}
+
+# Transform any set of strings to lowercase
+function to_lowercase {
+  echo "${@,,}"
+}
+
+# Transform any set of strings to uppercase
+function to_uppercase {
+  echo "${@^^}"
+}
+
+# Replace any variable separated with comma with space
+# e.g: DEBUG=foo,bar will become:
+# echo ${DEBUG//,/ }
+# foo bar
+function comma_to_space {
+  echo "${@//,/ }"
+}
+
+# Get based distro by discovering the package manager
+function get_package_manager {
+  if is_available rpm; then
+    OS_VENDOR=redhat
+  elif is_available dpkg; then
+    OS_VENDOR=ubuntu
+  fi
+}
+
+# Determine if current distribution is an Ubuntu-based distribution
+function is_ubuntu {
+  get_package_manager
+  [[ "$OS_VENDOR" == "ubuntu" ]]
+}
+
+# Determine if current distribution is a RedHat-based distribution
+function is_redhat {
+  get_package_manager
+  [[ "$OS_VENDOR" == "redhat" ]]
+}
+
+# Wait for a file to exist, regardless of the type
+function wait_for_file {
+  timeout 10 bash -c "while [ ! -e ${1} ]; do echo 'Waiting for ${1} to show up' && sleep 1 ; done"
+}
+
+function valid_scenarios {
+  log "Valid values for CEPH_DAEMON are $(to_uppercase $ALL_SCENARIOS)."
+  log "Valid values for the daemon parameter are $ALL_SCENARIOS"
+}
+
+function invalid_ceph_daemon {
+  if [ -z "$CEPH_DAEMON" ]; then
+    log "ERROR- One of CEPH_DAEMON or a daemon parameter must be defined as the name of the daemon you want to deploy."
+    valid_scenarios
+    exit 1
+  else
+    log "ERROR- unrecognized scenario."
+    valid_scenarios
+  fi
+}
+
+function get_osd_path {
+  echo "$OSD_PATH_BASE-$1/"
+}

ceph/templates/bin/_entrypoint.sh.tpl

@@ -0,0 +1,136 @@
+#!/bin/bash
+set -ex
+export LC_ALL=C
+
+source variables_entrypoint.sh
+source common_functions.sh
+source debug.sh
+
+###########################
+# CONFIGURATION GENERATOR #
+###########################
+
+# Load in the bootstrapping routines
+# based on the data store
+case "$KV_TYPE" in
+  etcd)
+    # TAG: kv_type_etcd
+    source /config.kv.etcd.sh
+    ;;
+  k8s|kubernetes)
+    # TAG: kv_type_k8s
+    source /config.k8s.sh
+    ;;
+  *)
+    source /config.static.sh
+    ;;
+esac
+
+
+###############
+# CEPH_DAEMON #
+###############
+
+# Normalize DAEMON to lowercase
+CEPH_DAEMON=$(to_lowercase ${CEPH_DAEMON})
+
+create_mandatory_directories
+
+# If we are given a valid first argument, set the
+# CEPH_DAEMON variable from it
+case "$CEPH_DAEMON" in
+  populate_kvstore)
+    # TAG: populate_kvstore
+    source populate_kv.sh
+    populate_kv
+    ;;
+  mon)
+    # TAG: mon
+    source start_mon.sh
+    start_mon
+    ;;
+  osd)
+    # TAG: osd
+    source start_osd.sh
+    start_osd
+    ;;
+  osd_directory)
+    # TAG: osd_directory
+    source start_osd.sh
+    OSD_TYPE="directory"
+    start_osd
+    ;;
+  osd_directory_single)
+    # TAG: osd_directory_single
+    source start_osd.sh
+    OSD_TYPE="directory_single"
+    start_osd
+    ;;
+  osd_ceph_disk)
+    # TAG: osd_ceph_disk
+    source start_osd.sh
+    OSD_TYPE="disk"
+    start_osd
+    ;;
+  osd_ceph_disk_prepare)
+    # TAG: osd_ceph_disk_prepare
+    source start_osd.sh
+    OSD_TYPE="prepare"
+    start_osd
+    ;;
+  osd_ceph_disk_activate)
+    # TAG: osd_ceph_disk_activate
+    source start_osd.sh
+    OSD_TYPE="activate"
+    start_osd
+    ;;
+  osd_ceph_activate_journal)
+    # TAG: osd_ceph_activate_journal
+    source start_osd.sh
+    OSD_TYPE="activate_journal"
+    start_osd
+    ;;
+  mds)
+    # TAG: mds
+    source start_mds.sh
+    start_mds
+    ;;
+  rgw)
+    # TAG: rgw
+    source start_rgw.sh
+    start_rgw
+    ;;
+  rgw_user)
+    # TAG: rgw_user
+    source start_rgw.sh
+    create_rgw_user
+    ;;
+  restapi)
+    # TAG: restapi
+    source start_restapi.sh
+    start_restapi
+    ;;
+  nfs)
+    # TAG: nfs
+    echo "Temporarily disabled due to broken package dependencies with nfs-ganesha"
+    echo "For more info see: https://github.com/ceph/ceph-docker/pull/564"
+    exit 1
+    source start_nfs.sh
+    start_nfs
+    ;;
+  zap_device)
+    # TAG: zap_device
+    source zap_device.sh
+    zap_device
+    ;;
+  mon_health)
+    # TAG: mon_health
+    source watch_mon_health.sh
+    watch_mon_health
+    ;;
+  *)
+    invalid_ceph_daemon
+  ;;
+esac
+
+exit 0

ceph/templates/bin/_osd_activate_journal.sh.tpl

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -ex
+
+function osd_activate_journal {
+  if [[ -z "${OSD_JOURNAL}" ]];then
+    log "ERROR- You must provide a device to build your OSD journal ie: /dev/sdb2"
+    exit 1
+  fi
+
+  # watch the udev event queue, and exit if all current events are handled
+  udevadm settle --timeout=600
+
+  # wait till partition exists
+  wait_for ${OSD_JOURNAL}
+
+  chown ceph. ${OSD_JOURNAL}
+  ceph-disk -v --setuser ceph --setgroup disk activate-journal ${OSD_JOURNAL}
+
+  start_osd
+}

ceph/templates/bin/_osd_common.sh.tpl

@@ -0,0 +1,31 @@
+# Start the latest OSD
+# In case of forego, we don't run ceph-osd, start_forego will do it later
+function start_osd() {
+  mode=$1 #forego or empty
+
+  OSD_ID=$(cat /var/lib/ceph/osd/$(ls -ltr /var/lib/ceph/osd/ | tail -n1 | awk -v pattern="$CLUSTER" '$0 ~ pattern {print $9}')/whoami)
+  OSD_PATH=$(get_osd_path $OSD_ID)
+  OSD_KEYRING="$OSD_PATH/keyring"
+  OSD_WEIGHT=$(df -P -k $OSD_PATH | tail -1 | awk '{ d= $2/1073741824 ; r = sprintf("%.2f", d); print r }')
+  ceph ${CLI_OPTS} --name=osd.${OSD_ID} --keyring=$OSD_KEYRING osd crush create-or-move -- ${OSD_ID} ${OSD_WEIGHT} ${CRUSH_LOCATION}
+
+  # ceph-disk activiate has exec'ed /usr/bin/ceph-osd ${CLI_OPTS} -f -i ${OSD_ID}
+  # wait till docker stop or ceph-osd is killed
+  OSD_PID=$(ps -ef |grep ceph-osd |grep osd.${OSD_ID} |awk '{print $2}')
+  if [ -n "${OSD_PID}" ]; then
+      log "OSD (PID ${OSD_PID}) is running, waiting till it exits"
+      while [ -e /proc/${OSD_PID} ]; do sleep 1;done
+  fi
+
+  if [[ "$mode" == "forego" ]]; then
+   echo "${CLUSTER}-${OSD_ID}: /usr/bin/ceph-osd ${CLI_OPTS} -f -i ${OSD_ID} --setuser ceph --setgroup disk" | tee -a /etc/forego/${CLUSTER}/Procfile
+  else
+   log "SUCCESS"
+   exec /usr/bin/ceph-osd ${CLI_OPTS} -f -i ${OSD_ID} --setuser ceph --setgroup disk
+  fi
+}
+
+# Starting forego
+function start_forego() {
+   exec /usr/local/bin/forego start -f /etc/forego/${CLUSTER}/Procfile
+}

ceph/templates/bin/_osd_directory.sh.tpl

@@ -0,0 +1,77 @@
+#!/bin/bash
+set -ex
+
+function osd_directory {
+  if [[ ! -d /var/lib/ceph/osd ]]; then
+    log "ERROR- could not find the osd directory, did you bind mount the OSD data directory?"
+    log "ERROR- use -v <host_osd_data_dir>:/var/lib/ceph/osd"
+    exit 1
+  fi
+
+  if [ -z "${HOSTNAME}" ]; then
+    log "HOSTNAME not set; This will prevent to add an OSD into the CRUSH map"
+    exit 1
+  fi
+
+  # check if anything is present, if not, create an osd and its directory
+  if [[ -n "$(find /var/lib/ceph/osd -prune -empty)" ]]; then
+    log "Creating osd with ceph --cluster ${CLUSTER} osd create"
+    OSD_ID=$(ceph --cluster ${CLUSTER} osd create)
+    if is_integer "$OSD_ID"; then
+      log "OSD created with ID: ${OSD_ID}"
+    else
+      log "OSD creation failed: ${OSD_ID}"
+      exit 1
+    fi
+
+    OSD_PATH=$(get_osd_path $OSD_ID)
+
+    # create the folder and own it
+    mkdir -p $OSD_PATH
+    chown ceph. $OSD_PATH
+    log "created folder $OSD_PATH"
+  fi
+
+  # create the directory and an empty Procfile
+  mkdir -p /etc/forego/${CLUSTER}
+  echo "" > /etc/forego/${CLUSTER}/Procfile
+
+  for OSD_ID in $(ls /var/lib/ceph/osd | sed 's/.*-//'); do
+    OSD_PATH=$(get_osd_path $OSD_ID)
+    OSD_KEYRING="$OSD_PATH/keyring"
+
+    if [ -n "${JOURNAL_DIR}" ]; then
+       OSD_J="${JOURNAL_DIR}/journal.${OSD_ID}"
+       chown -R ceph. ${JOURNAL_DIR}
+    else
+       if [ -n "${JOURNAL}" ]; then
+          OSD_J=${JOURNAL}
+          chown -R ceph. $(dirname ${JOURNAL_DIR})
+       else
+          OSD_J=${OSD_PATH}/journal
+       fi
+    fi
+    # check to see if our osd has been initialized
+    if [ ! -e ${OSD_PATH}/keyring ]; then
+      chown ceph. $OSD_PATH
+      # create osd key and file structure
+      ceph-osd ${CLI_OPTS} -i $OSD_ID --mkfs --mkkey --mkjournal --osd-journal ${OSD_J} --setuser ceph --setgroup ceph
+      if [ ! -e $OSD_BOOTSTRAP_KEYRING  ]; then
+        log "ERROR- $OSD_BOOTSTRAP_KEYRING must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o $OSD_BOOTSTRAP_KEYRING '"
+        exit 1
+      fi
+      timeout 10 ceph ${CLI_OPTS} --name client.bootstrap-osd --keyring $OSD_BOOTSTRAP_KEYRING health || exit 1
+      # add the osd key
+      ceph ${CLI_OPTS} --name client.bootstrap-osd --keyring $OSD_BOOTSTRAP_KEYRING auth add osd.${OSD_ID} -i ${OSD_KEYRING} osd 'allow *' mon 'allow profile osd'  || log $1
+      log "done adding key"
+      chown ceph. ${OSD_KEYRING}
+      chmod 0600 ${OSD_KEYRING}
+      # add the osd to the crush map
+      OSD_WEIGHT=$(df -P -k $OSD_PATH | tail -1 | awk '{ d= $2/1073741824 ; r = sprintf("%.2f", d); print r }')
+      ceph ${CLI_OPTS} --name=osd.${OSD_ID} --keyring=${OSD_KEYRING} osd crush create-or-move -- ${OSD_ID} ${OSD_WEIGHT} ${CRUSH_LOCATION}
+    fi
+    echo "${CLUSTER}-${OSD_ID}: /usr/bin/ceph-osd ${CLI_OPTS} -f -i ${OSD_ID} --osd-journal ${OSD_J} -k $OSD_KEYRING" | tee -a /etc/forego/${CLUSTER}/Procfile
+  done
+  log "SUCCESS"
+  start_forego
+}

ceph/templates/bin/_osd_directory_single.sh.tpl

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -ex
+
+function osd_directory_single {
+  if [[ ! -d /var/lib/ceph/osd ]]; then
+    log "ERROR- could not find the osd directory, did you bind mount the OSD data directory?"
+    log "ERROR- use -v <host_osd_data_dir>:/var/lib/ceph/osd"
+    exit 1
+  fi
+
+  # pick one osd and make sure no lock is held
+  for OSD_ID in $(ls /var/lib/ceph/osd | sed 's/.*-//'); do
+    OSD_PATH=$(get_osd_path $OSD_ID)
+    OSD_KEYRING="$OSD_PATH/keyring"
+
+    if [[ -n "$(find $OSD_PATH -prune -empty)" ]]; then
+      log "Looks like OSD: ${OSD_ID} has not been bootstrapped yet, doing nothing, moving on to the next discoverable OSD"
+    else
+      # check if the osd has a lock, if yes moving on, if not we run it
+      # many thanks to Julien Danjou for the python piece
+      if python -c "import sys, fcntl, struct; l = fcntl.fcntl(open('${OSD_PATH}/fsid', 'a'), fcntl.F_GETLK, struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 0, 0, 0)); l_type, l_whence, l_start, l_len, l_pid, l_sysid = struct.unpack('hhllhh', l); sys.exit(0 if l_type == fcntl.F_UNLCK else 1)"; then
+        log "Looks like OSD: ${OSD_ID} is not started, starting it..."
+        log "SUCCESS"
+        exec ceph-osd $DAEMON_OPTS -i ${OSD_ID} -k $OSD_KEYRING
+        break
+      fi
+    fi
+  done
+  log "Looks like all the OSDs are already running, doing nothing"
+  log "Exiting the container"
+  log "SUCCESS"
+  exit 0
+}

ceph/templates/bin/_osd_disk_activate.sh.tpl

@@ -0,0 +1,51 @@
+#!/bin/bash
+set -ex
+
+function osd_activate {
+  if [[ -z "${OSD_DEVICE}" ]];then
+    log "ERROR- You must provide a device to build your OSD ie: /dev/sdb"
+    exit 1
+  fi
+
+  CEPH_DISK_OPTIONS=""
+  DATA_UUID=$(blkid -o value -s PARTUUID ${OSD_DEVICE}1)
+  LOCKBOX_UUID=$(blkid -o value -s PARTUUID ${OSD_DEVICE}3 || true)
+  JOURNAL_PART=$(dev_part ${OSD_DEVICE} 2)
+  ACTUAL_OSD_DEVICE=$(readlink -f ${OSD_DEVICE}) # resolve /dev/disk/by-* names
+
+  # watch the udev event queue, and exit if all current events are handled
+  udevadm settle --timeout=600
+
+  # wait till partition exists then activate it
+  if [[ -n "${OSD_JOURNAL}" ]]; then
+    wait_for_file ${OSD_DEVICE}
+    chown ceph. ${OSD_JOURNAL}
+  else
+    wait_for_file $(dev_part ${OSD_DEVICE} 1)
+    chown ceph. $JOURNAL_PART
+  fi
+
+  DATA_PART=$(dev_part ${OSD_DEVICE} 1)
+  MOUNTED_PART=${DATA_PART}
+
+  if [[ ${OSD_DMCRYPT} -eq 1 ]]; then
+    echo "Mounting LOCKBOX directory"
+    # NOTE(leseb): adding || true so when this bug will be fixed the entrypoint will not fail
+    # Ceph bug tracker: http://tracker.ceph.com/issues/18945
+    mkdir -p /var/lib/ceph/osd-lockbox/${DATA_UUID}
+    mount /dev/disk/by-partuuid/${LOCKBOX_UUID} /var/lib/ceph/osd-lockbox/${DATA_UUID} || true
+    CEPH_DISK_OPTIONS="$CEPH_DISK_OPTIONS --dmcrypt"
+    MOUNTED_PART="/dev/mapper/${DATA_UUID}"
+  fi
+
+  ceph-disk -v --setuser ceph --setgroup disk activate ${CEPH_DISK_OPTIONS} --no-start-daemon ${DATA_PART}
+
+  OSD_ID=$(grep "${MOUNTED_PART}" /proc/mounts | awk '{print $2}' | grep -oh '[0-9]*')
+  OSD_PATH=$(get_osd_path $OSD_ID)
+  OSD_KEYRING="$OSD_PATH/keyring"
+  OSD_WEIGHT=$(df -P -k $OSD_PATH | tail -1 | awk '{ d= $2/1073741824 ; r = sprintf("%.2f", d); print r }')
+  ceph ${CLI_OPTS} --name=osd.${OSD_ID} --keyring=$OSD_KEYRING osd crush create-or-move -- ${OSD_ID} ${OSD_WEIGHT} ${CRUSH_LOCATION}
+
+  log "SUCCESS"
+  exec /usr/bin/ceph-osd ${CLI_OPTS} -f -i ${OSD_ID} --setuser ceph --setgroup disk
+}

ceph/templates/bin/_osd_disk_prepare.sh.tpl

@@ -0,0 +1,74 @@
+#!/bin/bash
+set -ex
+
+function osd_disk_prepare {
+  if [[ -z "${OSD_DEVICE}" ]];then
+    log "ERROR- You must provide a device to build your OSD ie: /dev/sdb"
+    exit 1
+  fi
+
+  if [[ ! -e "${OSD_DEVICE}" ]]; then
+    log "ERROR- The device pointed by OSD_DEVICE ($OSD_DEVICE) doesn't exist !"
+    exit 1
+  fi
+
+  if [ ! -e $OSD_BOOTSTRAP_KEYRING ]; then
+    log "ERROR- $OSD_BOOTSTRAP_KEYRING must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o $OSD_BOOTSTRAP_KEYRING'"
+    exit 1
+  fi
+  timeout 10 ceph ${CLI_OPTS} --name client.bootstrap-osd --keyring $OSD_BOOTSTRAP_KEYRING health || exit 1
+
+  # check device status first
+  if ! parted --script ${OSD_DEVICE} print > /dev/null 2>&1; then
+    if [[ ${OSD_FORCE_ZAP} -eq 1 ]]; then
+      log "It looks like ${OSD_DEVICE} isn't consistent, however OSD_FORCE_ZAP is enabled so we are zapping the device anyway"
+      ceph-disk -v zap ${OSD_DEVICE}
+    else
+      log "Regarding parted, device ${OSD_DEVICE} is inconsistent/broken/weird."
+      log "It would be too dangerous to destroy it without any notification."
+      log "Please set OSD_FORCE_ZAP to '1' if you really want to zap this disk."
+      exit 1
+    fi
+  fi
+
+  # then search for some ceph metadata on the disk
+  if [[ "$(parted --script ${OSD_DEVICE} print | egrep '^ 1.*ceph data')" ]]; then
+    if [[ ${OSD_FORCE_ZAP} -eq 1 ]]; then
+      log "It looks like ${OSD_DEVICE} is an OSD, however OSD_FORCE_ZAP is enabled so we are zapping the device anyway"
+      ceph-disk -v zap ${OSD_DEVICE}
+    else
+      log "INFO- It looks like ${OSD_DEVICE} is an OSD, set OSD_FORCE_ZAP=1 to use this device anyway and zap its content"
+      log "You can also use the zap_device scenario on the appropriate device to zap it"
+      log "Moving on, trying to activate the OSD now."
+      return
+    fi
+  fi
+
+  if [[ ${OSD_BLUESTORE} -eq 1 ]]; then
+    ceph-disk -v prepare ${CLI_OPTS} --bluestore ${OSD_DEVICE}
+  elif [[ ${OSD_DMCRYPT} -eq 1 ]]; then
+    get_admin_key
+    check_admin_key
+    # the admin key must be present on the node
+    # in order to store the encrypted key in the monitor's k/v store
+    ceph-disk -v prepare ${CLI_OPTS} --journal-uuid ${OSD_JOURNAL_UUID} --lockbox-uuid ${OSD_LOCKBOX_UUID} --dmcrypt ${OSD_DEVICE} ${OSD_JOURNAL}
+    echo "Unmounting LOCKBOX directory"
+    # NOTE(leseb): adding || true so when this bug will be fixed the entrypoint will not fail
+    # Ceph bug tracker: http://tracker.ceph.com/issues/18944
+    DATA_UUID=$(blkid -o value -s PARTUUID ${OSD_DEVICE}1)
+    umount /var/lib/ceph/osd-lockbox/${DATA_UUID} || true
+  else
+    ceph-disk -v prepare ${CLI_OPTS} --journal-uuid ${OSD_JOURNAL_UUID} ${OSD_DEVICE} ${OSD_JOURNAL}
+  fi
+
+  # watch the udev event queue, and exit if all current events are handled
+  udevadm settle --timeout=600
+
+  if [[ -n "${OSD_JOURNAL}" ]]; then
+    wait_for_file ${OSD_JOURNAL}
+    chown ceph. ${OSD_JOURNAL}
+  else
+    wait_for_file $(dev_part ${OSD_DEVICE} 2)
+    chown ceph. $(dev_part ${OSD_DEVICE} 2)
+  fi
+}

ceph/templates/bin/_osd_disks.sh.tpl

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -ex
+
+function osd_disks {
+  if [[ ! -d /var/lib/ceph/osd ]]; then
+    log "ERROR- could not find the osd directory, did you bind mount the OSD data directory?"
+    log "ERROR- use -v <host_osd_data_dir>:/var/lib/ceph/osd"
+    exit 1
+  fi
+  if [[  -z ${OSD_DISKS} ]]; then
+    log "ERROR- could not find the osd devices, did you configure OSD disks?"
+    log "ERROR- use -e OSD_DISKS=\"0:sdd 1:sde 2:sdf\""
+    exit 1
+  fi
+
+  # Create the directory and an empty Procfile
+  mkdir -p /etc/forego/${CLUSTER}
+  echo "" > /etc/forego/${CLUSTER}/Procfile
+
+  # check if anything is there, if not create an osd with directory
+  if [[ -z "$(find /var/lib/ceph/osd -prune -empty)" ]]; then
+    log "Mount existing and prepared OSD disks for ceph-cluster ${CLUSTER}"
+    for OSD_ID in $(ls /var/lib/ceph/osd | sed 's/.*-//'); do
+      OSD_PATH=$(get_osd_path $OSD_ID)
+      OSD_KEYRING="$OSD_PATH/keyring"
+      OSD_DEV=$(get_osd_dev ${OSD_ID})
+      if [[ -z ${OSD_DEV} ]]; then
+        log "No device mapping for ${CLUSTER}-${OSD_ID} for ceph-cluster ${CLUSTER}"
+        exit 1
+      fi
+      mount ${MOUNT_OPTS} $(dev_part ${OSD_DEV} 1) $OSD_PATH
+      xOSD_ID=$(cat $OSD_PATH/whoami)
+      if [[ "${OSD_ID}" != "${xOSD_ID}" ]]; then
+        log "Device ${OSD_DEV} is corrupt for $OSD_PATH"
+        exit 1
+      fi
+      echo "${CLUSTER}-${OSD_ID}: /usr/bin/ceph-osd ${CLI_OPTS} -f -i ${OSD_ID} --setuser ceph --setgroup disk" | tee -a /etc/forego/${CLUSTER}/Procfile
+    done
+    exec /usr/local/bin/forego start -f /etc/forego/${CLUSTER}/Procfile
+  fi
+
+  #
+  # As per the exec in the first statement, we only reach here if there is some OSDs
+  #
+  for OSD_DISK in ${OSD_DISKS}; do
+    OSD_DEV="/dev/$(echo ${OSD_DISK}|sed 's/\(.*\):\(.*\)/\2/')"
+
+    if [[ "$(parted --script ${OSD_DEV} print | egrep '^ 1.*ceph data')" ]]; then
+      if [[ ${OSD_FORCE_ZAP} -eq 1 ]]; then
+        ceph-disk -v zap ${OSD_DEV}
+      else
+        log "ERROR- It looks like the device ($OSD_DEV) is an OSD, set OSD_FORCE_ZAP=1 to use this device anyway and zap its content"
+        exit 1
+      fi
+    fi
+
+    ceph-disk -v prepare ${CLI_OPTS} ${OSD_DEV} ${OSD_JOURNAL}
+
+    # prepare the OSDs configuration and start them later
+    start_osd forego
+  done
+
+  log "SUCCESS"
+  # Actually, starting them as per forego configuration
+  start_forego
+}

ceph/templates/bin/_remove-mon.sh.tpl

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -ex
+
+ceph mon remove $(hostname -s)

ceph/templates/bin/_start_mds.sh.tpl

@@ -0,0 +1,58 @@
+#!/bin/bash
+set -ex
+
+function start_mds {
+  get_config
+  check_config
+
+  # Check to see if we are a new MDS
+  if [ ! -e $MDS_KEYRING ]; then
+
+    if [ -e $ADMIN_KEYRING ]; then
+       KEYRING_OPT="--name client.admin --keyring $ADMIN_KEYRING"
+    elif [ -e $MDS_BOOTSTRAP_KEYRING ]; then
+       KEYRING_OPT="--name client.bootstrap-mds --keyring $MDS_BOOTSTRAP_KEYRING"
+    else
+      log "ERROR- Failed to bootstrap MDS: could not find admin or bootstrap-mds keyring.  You can extract it from your current monitor by running 'ceph auth get client.bootstrap-mds -o $MDS_BOOTSTRAP_KEYRING"
+      exit 1
+    fi
+
+    timeout 10 ceph ${CLI_OPTS} $KEYRING_OPT health || exit 1
+
+    # Generate the MDS key
+    ceph ${CLI_OPTS} $KEYRING_OPT auth get-or-create mds.$MDS_NAME osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o $MDS_KEYRING
+    chown ceph. $MDS_KEYRING
+    chmod 600 $MDS_KEYRING
+
+  fi
+
+  # NOTE (leseb): having the admin keyring is really a security issue
+  # If we need to bootstrap a MDS we should probably create the following on the monitors
+  # I understand that this handy to do this here
+  # but having the admin key inside every container is a concern
+
+  # Create the Ceph filesystem, if necessary
+  if [ $CEPHFS_CREATE -eq 1 ]; then
+
+    get_admin_key
+    check_admin_key
+
+    if [[ "$(ceph ${CLI_OPTS} fs ls | grep -c name:.${CEPHFS_NAME},)" -eq 0 ]]; then
+       # Make sure the specified data pool exists
+       if ! ceph ${CLI_OPTS} osd pool stats ${CEPHFS_DATA_POOL} > /dev/null 2>&1; then
+          ceph ${CLI_OPTS} osd pool create ${CEPHFS_DATA_POOL} ${CEPHFS_DATA_POOL_PG}
+       fi
+
+       # Make sure the specified metadata pool exists
+       if ! ceph ${CLI_OPTS} osd pool stats ${CEPHFS_METADATA_POOL} > /dev/null 2>&1; then
+          ceph ${CLI_OPTS} osd pool create ${CEPHFS_METADATA_POOL} ${CEPHFS_METADATA_POOL_PG}
+       fi
+
+       ceph ${CLI_OPTS} fs new ${CEPHFS_NAME} ${CEPHFS_METADATA_POOL} ${CEPHFS_DATA_POOL}
+    fi
+  fi
+
+  log "SUCCESS"
+  # NOTE: prefixing this with exec causes it to die (commit suicide)
+  /usr/bin/ceph-mds $DAEMON_OPTS -i ${MDS_NAME}
+}

ceph/templates/bin/_start_mon.sh.tpl

@@ -0,0 +1,141 @@
+#!/bin/bash
+set -ex
+
+IPV4_REGEXP='[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}'
+IPV4_NETWORK_REGEXP="$IPV4_REGEXP/[0-9]\{1,2\}"
+
+function flat_to_ipv6 {
+  # Get a flat input like fe800000000000000042acfffe110003 and output fe80::0042:acff:fe11:0003
+  # This input usually comes from the ipv6_route or if_inet6 files from /proc
+
+  # First, split the string in set of 4 bytes with ":" as separator
+  value=$(echo "$@" | sed -e 's/.\{4\}/&:/g' -e '$s/\:$//')
+
+  # Let's remove the useless 0000 and "::"
+  value=${value//0000/:};
+  while $(echo $value | grep -q ":::"); do
+    value=${value//::/:};
+  done
+  echo $value
+}
+
+function get_ip {
+  NIC=$1
+  # IPv4 is the default unless we specify it
+  IP_VERSION=${2:-4}
+  # We should avoid reporting any IPv6 "scope local" interface that would make the ceph bind() call to fail
+  if is_available ip; then
+    ip -$IP_VERSION -o a s $NIC | grep "scope global" | awk '{ sub ("/..", "", $4); print $4 }' || true
+  else
+    case "$IP_VERSION" in
+      6)
+        # We don't want local scope, so let's remove field 4 if not 00
+        ip=$(flat_to_ipv6 $(grep $NIC /proc/net/if_inet6 | awk '$4==00 {print $1}'))
+        # IPv6 IPs should be surrounded by brackets to let ceph-monmap being happy
+        echo "[$ip]"
+        ;;
+      *)
+        grep -o "$IPV4_REGEXP" /proc/net/fib_trie | grep -vEw "^127|255$|0$" | head -1
+        ;;
+    esac
+  fi
+}
+
+function get_network {
+  NIC=$1
+  # IPv4 is the default unless we specify it
+  IP_VERSION=${2:-4}
+
+  case "$IP_VERSION" in
+    6)
+      if is_available ip; then
+        ip -$IP_VERSION route show dev $NIC | grep proto | awk '{ print $1 }' | grep -v default | grep -vi ^fe80 || true
+      else
+        # We don't want the link local routes
+        line=$(grep $NIC /proc/1/task/1/net/ipv6_route | awk '$2==40' | grep -v ^fe80 || true)
+        base=$(echo $line | awk '{ print $1 }')
+        base=$(flat_to_ipv6 $base)
+        mask=$(echo $line | awk '{ print $2 }')
+        echo "$base/$((16#$mask))"
+      fi
+      ;;
+    *)
+      if is_available ip; then
+        ip -$IP_VERSION route show dev $NIC | grep proto | awk '{ print $1 }' | grep -v default | grep "/" || true
+      else
+        grep -o "$IPV4_NETWORK_REGEXP" /proc/net/fib_trie | grep -vE "^127|^0" | head -1
+      fi
+      ;;
+  esac
+}
+
+function start_mon {
+  if [[ ${NETWORK_AUTO_DETECT} -eq 0 ]]; then
+      if [[ -z "$CEPH_PUBLIC_NETWORK" ]]; then
+        log "ERROR- CEPH_PUBLIC_NETWORK must be defined as the name of the network for the OSDs"
+        exit 1
+      fi
+
+      if [[ -z "$MON_IP" ]]; then
+        log "ERROR- MON_IP must be defined as the IP address of the monitor"
+        exit 1
+      fi
+  else
+    NIC_MORE_TRAFFIC=$(grep -vE "lo:|face|Inter" /proc/net/dev | sort -n -k 2 | tail -1 | awk '{ sub (":", "", $1); print $1 }')
+    IP_VERSION=4
+    if [ ${NETWORK_AUTO_DETECT} -gt 1 ]; then
+      MON_IP=$(get_ip ${NIC_MORE_TRAFFIC} ${NETWORK_AUTO_DETECT})
+      CEPH_PUBLIC_NETWORK=$(get_network ${NIC_MORE_TRAFFIC} ${NETWORK_AUTO_DETECT})
+      IP_VERSION=${NETWORK_AUTO_DETECT}
+    else # Means -eq 1
+      MON_IP="[$(get_ip ${NIC_MORE_TRAFFIC} 6)]"
+      CEPH_PUBLIC_NETWORK=$(get_network ${NIC_MORE_TRAFFIC} 6)
+      IP_VERSION=6
+      if [ -z "$MON_IP" ]; then
+        MON_IP=$(get_ip ${NIC_MORE_TRAFFIC})
+        CEPH_PUBLIC_NETWORK=$(get_network ${NIC_MORE_TRAFFIC})
+        IP_VERSION=4
+      fi
+    fi
+  fi
+
+  if [[ -z "$MON_IP" || -z "$CEPH_PUBLIC_NETWORK" ]]; then
+    log "ERROR- it looks like we have not been able to discover the network settings"
+    exit 1
+  fi
+
+  get_mon_config $IP_VERSION
+
+  # If we don't have a monitor keyring, this is a new monitor
+  if [ ! -e "$MON_DATA_DIR/keyring" ]; then
+    if [ ! -e $MON_KEYRING ]; then
+      log "ERROR- $MON_KEYRING must exist.  You can extract it from your current monitor by running 'ceph auth get mon. -o $MON_KEYRING' or use a KV Store"
+      exit 1
+    fi
+
+    if [ ! -e $MONMAP ]; then
+      log "ERROR- $MONMAP must exist.  You can extract it from your current monitor by running 'ceph mon getmap -o $MONMAP' or use a KV Store"
+      exit 1
+    fi
+
+    # Testing if it's not the first monitor, if one key doesn't exist we assume none of them exist
+    for keyring in $OSD_BOOTSTRAP_KEYRING $MDS_BOOTSTRAP_KEYRING $RGW_BOOTSTRAP_KEYRING $ADMIN_KEYRING; do
+      ceph-authtool $MON_KEYRING --import-keyring $keyring
+    done
+
+    # Prepare the monitor daemon's directory with the map and keyring
+    ceph-mon --setuser ceph --setgroup ceph --cluster ${CLUSTER} --mkfs -i ${MON_NAME} --inject-monmap $MONMAP --keyring $MON_KEYRING --mon-data "$MON_DATA_DIR"
+  else
+    log "Trying to get the most recent monmap..."
+    # Ignore when we timeout, in most cases that means the cluster has no quorum or
+    # no mons are up and running yet
+    timeout 5 ceph ${CLI_OPTS} mon getmap -o $MONMAP || true
+    ceph-mon --setuser ceph --setgroup ceph --cluster ${CLUSTER} -i ${MON_NAME} --inject-monmap $MONMAP --keyring $MON_KEYRING --mon-data "$MON_DATA_DIR"
+    timeout 7 ceph ${CLI_OPTS} mon add "${MON_NAME}" "${MON_IP}:6789" || true
+  fi
+
+  log "SUCCESS"
+
+  # start MON
+  exec /usr/bin/ceph-mon $DAEMON_OPTS -i ${MON_NAME} --mon-data "$MON_DATA_DIR" --public-addr "${MON_IP}:6789"
+}

ceph/templates/bin/_start_osd.sh.tpl

@@ -0,0 +1,61 @@
+#!/bin/bash
+set -ex
+
+if is_redhat; then
+  source /etc/sysconfig/ceph
+elif is_ubuntu; then
+  source /etc/default/ceph
+fi
+
+function start_osd {
+  get_config
+  check_config
+
+  if [ ${CEPH_GET_ADMIN_KEY} -eq 1 ]; then
+    get_admin_key
+    check_admin_key
+  fi
+
+  case "$OSD_TYPE" in
+    directory)
+      source osd_directory.sh
+      source osd_common.sh
+      osd_directory
+      ;;
+    directory_single)
+      source osd_directory_single.sh
+      osd_directory_single
+      ;;
+    disk)
+      osd_disk
+      ;;
+    prepare)
+      source osd_disk_prepare.sh
+      osd_disk_prepare
+      ;;
+    activate)
+      source osd_disk_activate.sh
+      osd_activate
+      ;;
+    devices)
+      source osd_disks.sh
+      source osd_common.sh
+      osd_disks
+      ;;
+    activate_journal)
+      source osd_activate_journal.sh
+      source osd_common.sh
+      osd_activate_journal
+      ;;
+    *)
+      osd_trying_to_determine_scenario
+      ;;
+  esac
+}
+
+function osd_disk {
+  source osd_disk_prepare.sh
+  source osd_disk_activate.sh
+  osd_disk_prepare
+  osd_activate
+}

ceph/templates/bin/_start_rgw.sh.tpl

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -ex
+
+function start_rgw {
+  get_config
+  check_config
+
+  if [ ${CEPH_GET_ADMIN_KEY} -eq 1 ]; then
+    get_admin_key
+    check_admin_key
+  fi
+
+  # Check to see if our RGW has been initialized
+  if [ ! -e $RGW_KEYRING ]; then
+
+    if [ ! -e $RGW_BOOTSTRAP_KEYRING ]; then
+      log "ERROR- $RGW_BOOTSTRAP_KEYRING must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-rgw -o $RGW_BOOTSTRAP_KEYRING'"
+      exit 1
+    fi
+
+    timeout 10 ceph ${CLI_OPTS} --name client.bootstrap-rgw --keyring $RGW_BOOTSTRAP_KEYRING health || exit 1
+
+    # Generate the RGW key
+    ceph ${CLI_OPTS} --name client.bootstrap-rgw --keyring $RGW_BOOTSTRAP_KEYRING auth get-or-create client.rgw.${RGW_NAME} osd 'allow rwx' mon 'allow rw' -o $RGW_KEYRING
+    chown ceph. $RGW_KEYRING
+    chmod 0600 $RGW_KEYRING
+  fi
+
+  log "SUCCESS"
+
+  RGW_FRONTENDS="civetweb port=$RGW_CIVETWEB_PORT"
+  if [ "$RGW_REMOTE_CGI" -eq 1 ]; then
+    RGW_FRONTENDS="fastcgi socket_port=$RGW_REMOTE_CGI_PORT socket_host=$RGW_REMOTE_CGI_HOST"
+  fi
+
+  exec /usr/bin/radosgw $DAEMON_OPTS -n client.rgw.${RGW_NAME} -k $RGW_KEYRING --rgw-socket-path="" --rgw-zonegroup="$RGW_ZONEGROUP" --rgw-zone="$RGW_ZONE" --rgw-frontends="$RGW_FRONTENDS"
+}
+
+function create_rgw_user {
+
+  # Check to see if our RGW has been initialized
+  if [ ! -e /var/lib/ceph/radosgw/keyring ]; then
+    log "ERROR- /var/lib/ceph/radosgw/keyring must exist. Please get it from your Rados Gateway"
+    exit 1
+  fi
+
+  mv /var/lib/ceph/radosgw/keyring $RGW_KEYRING
+
+  USER_KEY=""
+  if [ -n "${RGW_USER_SECRET_KEY}" ]; then
+    USER_KEY="--access-key=${RGW_USER_USER_KEY} --secret=${RGW_USER_SECRET_KEY}"
+  fi
+
+  exec radosgw-admin user create --uid=${RGW_USER} ${USER_KEY} --display-name="RGW ${RGW_USER} User" -c /etc/ceph/${CLUSTER}.conf
+}

ceph/templates/bin/_watch_mon_health.sh.tpl

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -ex
+
+function watch_mon_health {
+
+  while [ true ]
+  do
+    log "checking for zombie mons"
+    /check_zombie_mons.py || true
+    log "sleep 30 sec"
+    sleep 30
+  done
+}

ceph/templates/configmap-bin.yaml

@@ -33,3 +33,33 @@ data:
   ceph-namespace-client-key.sh: |+
 {{ tuple "bin/_ceph-namespace-client-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
+  common_functions.sh: |+
+{{ tuple "bin/_common_functions.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  entrypoint.sh: |+
+{{ tuple "bin/_entrypoint.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  osd_activate_journal.sh: |+
+{{ tuple "bin/_osd_activate_journal.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  osd_common.sh: |+
+{{ tuple "bin/_osd_common.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  osd_directory.sh: |+
+{{ tuple "bin/_osd_directory.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  osd_directory_single.sh: |+
+{{ tuple "bin/_osd_directory_single.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  osd_disk_activate.sh: |+
+{{ tuple "bin/_osd_disk_activate.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  osd_disk_prepare.sh: |+
+{{ tuple "bin/_osd_disk_prepare.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  osd_disks.sh: |+
+{{ tuple "bin/_osd_disks.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  remove-mon.sh: |+
+{{ tuple "bin/_remove-mon.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  start_mon.sh: |+
+{{ tuple "bin/_start_mon.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  start_osd.sh: |+
+{{ tuple "bin/_start_osd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  start_mds.sh: |+
+{{ tuple "bin/_start_mds.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  start_rgw.sh: |+
+{{ tuple "bin/_start_rgw.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  watch_mon_health.sh: |+
+{{ tuple "bin/_watch_mon_health.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

ceph/templates/daemonset-osd.yaml

@@ -78,6 +78,22 @@ spec:
             - name: ceph
               mountPath: /var/lib/ceph
               readOnly: false
+            - name: ceph-bin
+              mountPath: /entrypoint.sh
+              subPath: entrypoint.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /start_osd.sh
+              subPath: start_osd.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /osd_directory.sh
+              subPath: osd_directory.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /common_functions.sh
+              subPath: common_functions.sh
+              readOnly: true
             - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
@@ -111,6 +127,10 @@ spec:
         - name: ceph
           hostPath:
             path: {{ .Values.ceph.storage.var_directory }}
+        - name: ceph-bin
+          configMap:
+            name: ceph-bin
+            defaultMode: 0555
         - name: ceph-etc
           configMap:
             name: ceph-etc

ceph/templates/deployment-mds.yaml

@@ -64,6 +64,18 @@ spec:
           command:
             - /entrypoint.sh
           volumeMounts:
+            - name: ceph-bin
+              mountPath: /entrypoint.sh
+              subPath: entrypoint.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /start_mds.sh
+              subPath: start_mds.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /common_functions.sh
+              subPath: common_functions.sh
+              readOnly: true
             - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
@@ -102,6 +114,10 @@ spec:
           configMap:
             name: ceph-etc
             defaultMode: 0444
+        - name: ceph-bin
+          configMap:
+            name: ceph-bin
+            defaultMode: 0555
         - name: ceph-client-admin-keyring
           secret:
             secretName: {{ .Values.secrets.keyrings.admin }}

ceph/templates/deployment-moncheck.yaml

@@ -64,6 +64,18 @@ spec:
           command:
             - /entrypoint.sh
           volumeMounts:
+            - name: ceph-bin
+              mountPath: /entrypoint.sh
+              subPath: entrypoint.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /watch_mon_health.sh
+              subPath: watch_mon_health.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /common_functions.sh
+              subPath: common_functions.sh
+              readOnly: true
             - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
@@ -93,6 +105,10 @@ spec:
           configMap:
             name: ceph-etc
             defaultMode: 0444
+        - name: ceph-bin
+          configMap:
+            name: ceph-bin
+            defaultMode: 0555
         - name: ceph-client-admin-keyring
           secret:
             secretName: {{ .Values.secrets.keyrings.admin }}

ceph/templates/deployment-rgw.yaml

@@ -65,6 +65,18 @@ spec:
           command:
             - /entrypoint.sh
           volumeMounts:
+            - name: ceph-bin
+              mountPath: /entrypoint.sh
+              subPath: entrypoint.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /start_rgw.sh
+              subPath: start_rgw.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /common_functions.sh
+              subPath: common_functions.sh
+              readOnly: true
             - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
@@ -101,6 +113,10 @@ spec:
                 port: {{ .Values.network.port.rgw_target }}
               timeoutSeconds: 5
       volumes:
+        - name: ceph-bin
+          configMap:
+            name: ceph-bin
+            defaultMode: 0555
         - name: ceph-etc
           configMap:
             name: ceph-etc

ceph/templates/statefulset-mon.yaml

@@ -96,6 +96,22 @@ spec:
                   command:
                     - "/remove-mon.sh"
           volumeMounts:
+            - name: ceph-bin
+              mountPath: /entrypoint.sh
+              subPath: entrypoint.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /start_mon.sh
+              subPath: start_mon.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /remove-mon.sh
+              subPath: remove-mon.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /common_functions.sh
+              subPath: common_functions.sh
+              readOnly: true
             - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
@@ -130,6 +146,10 @@ spec:
                 port: 6789
               timeoutSeconds: 5
       volumes:
+        - name: ceph-bin
+          configMap:
+            name: ceph-bin
+            defaultMode: 0555
         - name: ceph-etc
           configMap:
             name: ceph-etc