From 2b42632c9b0f09fbd35ff6428be318d6319b336c Mon Sep 17 00:00:00 2001 From: Chinasubbareddy Mallavarapu Date: Mon, 18 Nov 2019 15:02:23 -0600 Subject: [PATCH] [ceph-osd] Separate ceph-disk based deployment scripts This is to create a different folder for ceph-disk based deplyoments so that it will be easy to maintain when we introduce ceph-volume. Separate folder for both the tools gives us flexibilty to develop or fix the issues and commit the code to respective folders without breaking other tool-based deployments. Change-Id: Ib0099d292a8692dc6676eb5ed624d5d1ef677cfe --- ceph-osd/templates/bin/osd/_directory.sh.tpl | 2 +- ceph-osd/templates/bin/osd/_init.sh.tpl | 209 +--------------- ceph-osd/templates/bin/osd/_start.sh.tpl | 2 +- .../bin/osd/{ => ceph-disk}/_block.sh.tpl | 2 +- .../bin/osd/{ => ceph-disk}/_bluestore.sh.tpl | 2 +- .../bin/osd/{ => ceph-disk}/_common.sh.tpl | 0 .../osd/ceph-disk/_init-with-ceph-disk.sh.tpl | 227 ++++++++++++++++++ ceph-osd/templates/configmap-bin.yaml | 16 +- ceph-osd/templates/daemonset-osd.yaml | 32 +-- ceph-osd/values.yaml | 3 + 10 files changed, 263 insertions(+), 232 deletions(-) rename ceph-osd/templates/bin/osd/{ => ceph-disk}/_block.sh.tpl (98%) rename ceph-osd/templates/bin/osd/{ => ceph-disk}/_bluestore.sh.tpl (98%) rename ceph-osd/templates/bin/osd/{ => ceph-disk}/_common.sh.tpl (100%) create mode 100644 ceph-osd/templates/bin/osd/ceph-disk/_init-with-ceph-disk.sh.tpl diff --git a/ceph-osd/templates/bin/osd/_directory.sh.tpl b/ceph-osd/templates/bin/osd/_directory.sh.tpl index 255412c7f..74432ab14 100644 --- a/ceph-osd/templates/bin/osd/_directory.sh.tpl +++ b/ceph-osd/templates/bin/osd/_directory.sh.tpl @@ -19,7 +19,7 @@ limitations under the License. set -ex export LC_ALL=C -source /tmp/osd-common.sh +source /tmp/osd-common-ceph-disk.sh : "${JOURNAL_DIR:=/var/lib/ceph/journal}" diff --git a/ceph-osd/templates/bin/osd/_init.sh.tpl b/ceph-osd/templates/bin/osd/_init.sh.tpl index e0f549092..4564617f9 100644 --- a/ceph-osd/templates/bin/osd/_init.sh.tpl +++ b/ceph-osd/templates/bin/osd/_init.sh.tpl @@ -18,210 +18,5 @@ limitations under the License. set -ex -source /tmp/osd-common.sh - -: "${OSD_FORCE_REPAIR:=1}" -# We do not want to zap journal disk. Tracking this option seperatly. -: "${JOURNAL_FORCE_ZAP:=0}" - -if [ "x${STORAGE_TYPE%-*}" == "xbluestore" ]; then - export OSD_BLUESTORE=1 -fi - -if [ "x${STORAGE_TYPE%-*}" == "xdirectory" ]; then - export OSD_DEVICE="/var/lib/ceph/osd" -else - export OSD_DEVICE=$(readlink -f ${STORAGE_LOCATION}) -fi - -if [ "x$JOURNAL_TYPE" == "xdirectory" ]; then - export OSD_JOURNAL="/var/lib/ceph/journal" -else - export OSD_JOURNAL=$(readlink -f ${JOURNAL_LOCATION}) -fi - -function osd_disk_prepare { - if [[ -z "${OSD_DEVICE}" ]];then - echo "ERROR- You must provide a device to build your OSD ie: /dev/sdb" - exit 1 - fi - - if [[ ! -b "${OSD_DEVICE}" ]]; then - echo "ERROR- The device pointed by OSD_DEVICE ($OSD_DEVICE) doesn't exist !" - exit 1 - fi - - if [ ! -e $OSD_BOOTSTRAP_KEYRING ]; then - echo "ERROR- $OSD_BOOTSTRAP_KEYRING must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o $OSD_BOOTSTRAP_KEYRING'" - exit 1 - fi - timeout 10 ceph ${CLI_OPTS} --name client.bootstrap-osd --keyring $OSD_BOOTSTRAP_KEYRING health || exit 1 - - # check device status first - if ! parted --script ${OSD_DEVICE} print > /dev/null 2>&1; then - if [[ ${OSD_FORCE_REPAIR} -eq 1 ]]; then - echo "It looks like ${OSD_DEVICE} isn't consistent, however OSD_FORCE_REPAIR is enabled so we are zapping the device anyway" - disk_zap ${OSD_DEVICE} - else - echo "Regarding parted, device ${OSD_DEVICE} is inconsistent/broken/weird." - echo "It would be too dangerous to destroy it without any notification." - echo "Please set OSD_FORCE_REPAIR to '1' if you really want to zap this disk." - exit 1 - fi - fi - - # then search for some ceph metadata on the disk - if [[ "$(parted --script ${OSD_DEVICE} print | egrep '^ 1.*ceph data')" ]]; then - if [[ ${OSD_FORCE_REPAIR} -eq 1 ]]; then - if [ -b "${OSD_DEVICE}1" ]; then - local cephFSID=$(ceph-conf --lookup fsid) - if [ ! -z "${cephFSID}" ]; then - local tmpmnt=$(mktemp -d) - mount ${OSD_DEVICE}1 ${tmpmnt} - if [ "${OSD_BLUESTORE:-0}" -ne 1 ] && [ "x$JOURNAL_TYPE" != "xdirectory" ]; then - # we only care about journals for filestore. - if [ -f "${tmpmnt}/whoami" ]; then - OSD_JOURNAL_DISK=$(readlink -f "${tmpmnt}/journal") - local osd_id=$(cat "${tmpmnt}/whoami") - if [ ! -b "${OSD_JOURNAL_DISK}" ]; then - OSD_JOURNAL=$(readlink -f ${OSD_JOURNAL}) - local jdev=$(echo ${OSD_JOURNAL} | sed 's/[0-9]//g') - if [ ${jdev} == ${OSD_JOURNAL} ]; then - echo "It appears that ${OSD_DEVICE} is missing the journal at ${OSD_JOURNAL}." - echo "Because OSD_FORCE_REPAIR is set, we will wipe the metadata of the OSD and zap it." - rm -rf ${tmpmnt}/ceph_fsid - else - echo "It appears that ${OSD_DEVICE} is missing the journal at ${OSD_JOURNAL_DISK}." - echo "Because OSD_FORCE_REPAIR is set and paritions are manually defined, we will" - echo "attempt to recreate the missing journal device partitions." - osd_journal_create ${OSD_JOURNAL} - ln -sf /dev/disk/by-partuuid/${OSD_JOURNAL_UUID} ${tmpmnt}/journal - echo ${OSD_JOURNAL_UUID} | tee ${tmpmnt}/journal_uuid - chown ceph. ${OSD_JOURNAL} - # During OSD start we will format the journal and set the fsid - touch ${tmpmnt}/run_mkjournal - fi - fi - else - echo "It looks like ${OSD_DEVICE} has a ceph data partition but is missing it's metadata." - echo "The device may contain inconsistent metadata or be corrupted." - echo "Because OSD_FORCE_REPAIR is set, we will wipe the metadata of the OSD and zap it." - rm -rf ${tmpmnt}/ceph_fsid - fi - fi - if [ -f "${tmpmnt}/ceph_fsid" ]; then - osdFSID=$(cat "${tmpmnt}/ceph_fsid") - if [ ${osdFSID} != ${cephFSID} ]; then - echo "It looks like ${OSD_DEVICE} is an OSD belonging to a different (or old) ceph cluster." - echo "The OSD FSID is ${osdFSID} while this cluster is ${cephFSID}" - echo "Because OSD_FORCE_REPAIR was set, we will zap this device." - zap_extra_partitions ${tmpmnt} - umount ${tmpmnt} - disk_zap ${OSD_DEVICE} - else - umount ${tmpmnt} - echo "It looks like ${OSD_DEVICE} is an OSD belonging to a this ceph cluster." - echo "OSD_FORCE_REPAIR is set, but will be ignored and the device will not be zapped." - echo "Moving on, trying to activate the OSD now." - return - fi - else - echo "It looks like ${OSD_DEVICE} has a ceph data partition but no FSID." - echo "Because OSD_FORCE_REPAIR was set, we will zap this device." - zap_extra_partitions ${tmpmnt} - umount ${tmpmnt} - disk_zap ${OSD_DEVICE} - fi - else - echo "Unable to determine the FSID of the current cluster." - echo "OSD_FORCE_REPAIR is set, but this OSD will not be zapped." - echo "Moving on, trying to activate the OSD now." - return - fi - else - echo "parted says ${OSD_DEVICE}1 should exist, but we do not see it." - echo "We will ignore OSD_FORCE_REPAIR and try to use the device as-is" - echo "Moving on, trying to activate the OSD now." - return - fi - else - echo "INFO- It looks like ${OSD_DEVICE} is an OSD, set OSD_FORCE_REPAIR=1 to use this device anyway and zap its content" - echo "You can also use the disk_zap scenario on the appropriate device to zap it" - echo "Moving on, trying to activate the OSD now." - return - fi - fi - - if [ "${OSD_BLUESTORE:-0}" -eq 1 ]; then - CLI_OPTS="${CLI_OPTS} --bluestore" - - if [ ! -z "$BLOCK_DB" ]; then - CLI_OPTS="${CLI_OPTS} --block.db ${BLOCK_DB}" - fi - - if [ ! -z "$BLOCK_WAL" ]; then - CLI_OPTS="${CLI_OPTS} --block.wal ${BLOCK_WAL}" - fi - - CLI_OPTS="${CLI_OPTS} ${OSD_DEVICE}" - else - # we only care about journals for filestore. - osd_journal_prepare - - CLI_OPTS="${CLI_OPTS} --journal-uuid ${OSD_JOURNAL_UUID} ${OSD_DEVICE}" - - if [ "x$JOURNAL_TYPE" == "xdirectory" ]; then - CLI_OPTS="${CLI_OPTS} --journal-file" - else - CLI_OPTS="${CLI_OPTS} ${OSD_JOURNAL}" - fi - fi - - udev_settle - ceph-disk -v prepare ${CLI_OPTS} -} - -function osd_journal_create { - local osd_journal=${1} - local osd_journal_partition=$(echo ${osd_journal} | sed 's/[^0-9]//g') - local jdev=$(echo ${osd_journal} | sed 's/[0-9]//g') - if [ -b "${jdev}" ]; then - sgdisk --new=${osd_journal_partition}:0:+${OSD_JOURNAL_SIZE}M \ - --change-name='${osd_journal_partition}:ceph journal' \ - --partition-guid=${osd_journal_partition}:${OSD_JOURNAL_UUID} \ - --typecode=${osd_journal_partition}:45b0969e-9b03-4f30-b4c6-b4b80ceff106 --mbrtogpt -- ${jdev} - OSD_JOURNAL=$(dev_part ${jdev} ${osd_journal_partition}) - udev_settle - else - echo "The backing device ${jdev} for ${OSD_JOURNAL} does not exist on this system." - exit 1 - fi -} - -function osd_journal_prepare { - if [ -n "${OSD_JOURNAL}" ]; then - if [ -b ${OSD_JOURNAL} ]; then - OSD_JOURNAL=$(readlink -f ${OSD_JOURNAL}) - OSD_JOURNAL_PARTITION=$(echo ${OSD_JOURNAL} | sed 's/[^0-9]//g') - local jdev=$(echo ${OSD_JOURNAL} | sed 's/[0-9]//g') - if [ -z "${OSD_JOURNAL_PARTITION}" ]; then - OSD_JOURNAL=$(dev_part ${jdev} ${OSD_JOURNAL_PARTITION}) - else - OSD_JOURNAL=${OSD_JOURNAL} - fi - elif [ "x$JOURNAL_TYPE" != "xdirectory" ]; then - # The block device exists but doesn't appear to be paritioned, we will proceed with parititioning the device. - OSD_JOURNAL=$(readlink -f ${OSD_JOURNAL}) - osd_journal_create ${OSD_JOURNAL} - fi - chown ceph. ${OSD_JOURNAL} - elif [ "x$JOURNAL_TYPE" != "xdirectory" ]; then - echo "No journal device specified. OSD and journal will share ${OSD_DEVICE}" - echo "For better performance on HDD, consider moving your journal to a separate device" - fi - CLI_OPTS="${CLI_OPTS} --filestore" -} - -if ! [ "x${STORAGE_TYPE%-*}" == "xdirectory" ]; then - osd_disk_prepare -fi +echo "Initializing the osd with ${DEPLOY_TOOL}" +exec "/tmp/init-${DEPLOY_TOOL}.sh" diff --git a/ceph-osd/templates/bin/osd/_start.sh.tpl b/ceph-osd/templates/bin/osd/_start.sh.tpl index 7b0e7cf63..cf9280f8e 100644 --- a/ceph-osd/templates/bin/osd/_start.sh.tpl +++ b/ceph-osd/templates/bin/osd/_start.sh.tpl @@ -19,4 +19,4 @@ limitations under the License. set -ex echo "LAUNCHING OSD: in ${STORAGE_TYPE%-*}:${STORAGE_TYPE#*-} mode" -exec "/tmp/osd-${STORAGE_TYPE%-*}.sh" +exec "/tmp/osd-${STORAGE_TYPE%-*}-${DEPLOY_TOOL}.sh" diff --git a/ceph-osd/templates/bin/osd/_block.sh.tpl b/ceph-osd/templates/bin/osd/ceph-disk/_block.sh.tpl similarity index 98% rename from ceph-osd/templates/bin/osd/_block.sh.tpl rename to ceph-osd/templates/bin/osd/ceph-disk/_block.sh.tpl index aa40d68d0..0773e3168 100644 --- a/ceph-osd/templates/bin/osd/_block.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-disk/_block.sh.tpl @@ -16,7 +16,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -source /tmp/osd-common.sh +source /tmp/osd-common-ceph-disk.sh set -ex diff --git a/ceph-osd/templates/bin/osd/_bluestore.sh.tpl b/ceph-osd/templates/bin/osd/ceph-disk/_bluestore.sh.tpl similarity index 98% rename from ceph-osd/templates/bin/osd/_bluestore.sh.tpl rename to ceph-osd/templates/bin/osd/ceph-disk/_bluestore.sh.tpl index 69280c8f4..f51f7fa1b 100644 --- a/ceph-osd/templates/bin/osd/_bluestore.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-disk/_bluestore.sh.tpl @@ -16,7 +16,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -source /tmp/osd-common.sh +source /tmp/osd-common-ceph-disk.sh set -ex diff --git a/ceph-osd/templates/bin/osd/_common.sh.tpl b/ceph-osd/templates/bin/osd/ceph-disk/_common.sh.tpl similarity index 100% rename from ceph-osd/templates/bin/osd/_common.sh.tpl rename to ceph-osd/templates/bin/osd/ceph-disk/_common.sh.tpl diff --git a/ceph-osd/templates/bin/osd/ceph-disk/_init-with-ceph-disk.sh.tpl b/ceph-osd/templates/bin/osd/ceph-disk/_init-with-ceph-disk.sh.tpl new file mode 100644 index 000000000..19f087489 --- /dev/null +++ b/ceph-osd/templates/bin/osd/ceph-disk/_init-with-ceph-disk.sh.tpl @@ -0,0 +1,227 @@ +#!/bin/bash + +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +source /tmp/osd-common-ceph-disk.sh + +: "${OSD_FORCE_REPAIR:=1}" +# We do not want to zap journal disk. Tracking this option seperatly. +: "${JOURNAL_FORCE_ZAP:=0}" + +if [ "x${STORAGE_TYPE%-*}" == "xbluestore" ]; then + export OSD_BLUESTORE=1 +fi + +if [ "x${STORAGE_TYPE%-*}" == "xdirectory" ]; then + export OSD_DEVICE="/var/lib/ceph/osd" +else + export OSD_DEVICE=$(readlink -f ${STORAGE_LOCATION}) +fi + +if [ "x$JOURNAL_TYPE" == "xdirectory" ]; then + export OSD_JOURNAL="/var/lib/ceph/journal" +else + export OSD_JOURNAL=$(readlink -f ${JOURNAL_LOCATION}) +fi + +function osd_disk_prepare { + if [[ -z "${OSD_DEVICE}" ]];then + echo "ERROR- You must provide a device to build your OSD ie: /dev/sdb" + exit 1 + fi + + if [[ ! -b "${OSD_DEVICE}" ]]; then + echo "ERROR- The device pointed by OSD_DEVICE ($OSD_DEVICE) doesn't exist !" + exit 1 + fi + + if [ ! -e $OSD_BOOTSTRAP_KEYRING ]; then + echo "ERROR- $OSD_BOOTSTRAP_KEYRING must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o $OSD_BOOTSTRAP_KEYRING'" + exit 1 + fi + timeout 10 ceph ${CLI_OPTS} --name client.bootstrap-osd --keyring $OSD_BOOTSTRAP_KEYRING health || exit 1 + + # check device status first + if ! parted --script ${OSD_DEVICE} print > /dev/null 2>&1; then + if [[ ${OSD_FORCE_REPAIR} -eq 1 ]]; then + echo "It looks like ${OSD_DEVICE} isn't consistent, however OSD_FORCE_REPAIR is enabled so we are zapping the device anyway" + disk_zap ${OSD_DEVICE} + else + echo "Regarding parted, device ${OSD_DEVICE} is inconsistent/broken/weird." + echo "It would be too dangerous to destroy it without any notification." + echo "Please set OSD_FORCE_REPAIR to '1' if you really want to zap this disk." + exit 1 + fi + fi + + # then search for some ceph metadata on the disk + if [[ "$(parted --script ${OSD_DEVICE} print | egrep '^ 1.*ceph data')" ]]; then + if [[ ${OSD_FORCE_REPAIR} -eq 1 ]]; then + if [ -b "${OSD_DEVICE}1" ]; then + local cephFSID=$(ceph-conf --lookup fsid) + if [ ! -z "${cephFSID}" ]; then + local tmpmnt=$(mktemp -d) + mount ${OSD_DEVICE}1 ${tmpmnt} + if [ "${OSD_BLUESTORE:-0}" -ne 1 ] && [ "x$JOURNAL_TYPE" != "xdirectory" ]; then + # we only care about journals for filestore. + if [ -f "${tmpmnt}/whoami" ]; then + OSD_JOURNAL_DISK=$(readlink -f "${tmpmnt}/journal") + local osd_id=$(cat "${tmpmnt}/whoami") + if [ ! -b "${OSD_JOURNAL_DISK}" ]; then + OSD_JOURNAL=$(readlink -f ${OSD_JOURNAL}) + local jdev=$(echo ${OSD_JOURNAL} | sed 's/[0-9]//g') + if [ ${jdev} == ${OSD_JOURNAL} ]; then + echo "It appears that ${OSD_DEVICE} is missing the journal at ${OSD_JOURNAL}." + echo "Because OSD_FORCE_REPAIR is set, we will wipe the metadata of the OSD and zap it." + rm -rf ${tmpmnt}/ceph_fsid + else + echo "It appears that ${OSD_DEVICE} is missing the journal at ${OSD_JOURNAL_DISK}." + echo "Because OSD_FORCE_REPAIR is set and paritions are manually defined, we will" + echo "attempt to recreate the missing journal device partitions." + osd_journal_create ${OSD_JOURNAL} + ln -sf /dev/disk/by-partuuid/${OSD_JOURNAL_UUID} ${tmpmnt}/journal + echo ${OSD_JOURNAL_UUID} | tee ${tmpmnt}/journal_uuid + chown ceph. ${OSD_JOURNAL} + # During OSD start we will format the journal and set the fsid + touch ${tmpmnt}/run_mkjournal + fi + fi + else + echo "It looks like ${OSD_DEVICE} has a ceph data partition but is missing it's metadata." + echo "The device may contain inconsistent metadata or be corrupted." + echo "Because OSD_FORCE_REPAIR is set, we will wipe the metadata of the OSD and zap it." + rm -rf ${tmpmnt}/ceph_fsid + fi + fi + if [ -f "${tmpmnt}/ceph_fsid" ]; then + osdFSID=$(cat "${tmpmnt}/ceph_fsid") + if [ ${osdFSID} != ${cephFSID} ]; then + echo "It looks like ${OSD_DEVICE} is an OSD belonging to a different (or old) ceph cluster." + echo "The OSD FSID is ${osdFSID} while this cluster is ${cephFSID}" + echo "Because OSD_FORCE_REPAIR was set, we will zap this device." + zap_extra_partitions ${tmpmnt} + umount ${tmpmnt} + disk_zap ${OSD_DEVICE} + else + umount ${tmpmnt} + echo "It looks like ${OSD_DEVICE} is an OSD belonging to a this ceph cluster." + echo "OSD_FORCE_REPAIR is set, but will be ignored and the device will not be zapped." + echo "Moving on, trying to activate the OSD now." + return + fi + else + echo "It looks like ${OSD_DEVICE} has a ceph data partition but no FSID." + echo "Because OSD_FORCE_REPAIR was set, we will zap this device." + zap_extra_partitions ${tmpmnt} + umount ${tmpmnt} + disk_zap ${OSD_DEVICE} + fi + else + echo "Unable to determine the FSID of the current cluster." + echo "OSD_FORCE_REPAIR is set, but this OSD will not be zapped." + echo "Moving on, trying to activate the OSD now." + return + fi + else + echo "parted says ${OSD_DEVICE}1 should exist, but we do not see it." + echo "We will ignore OSD_FORCE_REPAIR and try to use the device as-is" + echo "Moving on, trying to activate the OSD now." + return + fi + else + echo "INFO- It looks like ${OSD_DEVICE} is an OSD, set OSD_FORCE_REPAIR=1 to use this device anyway and zap its content" + echo "You can also use the disk_zap scenario on the appropriate device to zap it" + echo "Moving on, trying to activate the OSD now." + return + fi + fi + + if [ "${OSD_BLUESTORE:-0}" -eq 1 ]; then + CLI_OPTS="${CLI_OPTS} --bluestore" + + if [ ! -z "$BLOCK_DB" ]; then + CLI_OPTS="${CLI_OPTS} --block.db ${BLOCK_DB}" + fi + + if [ ! -z "$BLOCK_WAL" ]; then + CLI_OPTS="${CLI_OPTS} --block.wal ${BLOCK_WAL}" + fi + + CLI_OPTS="${CLI_OPTS} ${OSD_DEVICE}" + else + # we only care about journals for filestore. + osd_journal_prepare + + CLI_OPTS="${CLI_OPTS} --journal-uuid ${OSD_JOURNAL_UUID} ${OSD_DEVICE}" + + if [ "x$JOURNAL_TYPE" == "xdirectory" ]; then + CLI_OPTS="${CLI_OPTS} --journal-file" + else + CLI_OPTS="${CLI_OPTS} ${OSD_JOURNAL}" + fi + fi + + udev_settle + ceph-disk -v prepare ${CLI_OPTS} +} + +function osd_journal_create { + local osd_journal=${1} + local osd_journal_partition=$(echo ${osd_journal} | sed 's/[^0-9]//g') + local jdev=$(echo ${osd_journal} | sed 's/[0-9]//g') + if [ -b "${jdev}" ]; then + sgdisk --new=${osd_journal_partition}:0:+${OSD_JOURNAL_SIZE}M \ + --change-name='${osd_journal_partition}:ceph journal' \ + --partition-guid=${osd_journal_partition}:${OSD_JOURNAL_UUID} \ + --typecode=${osd_journal_partition}:45b0969e-9b03-4f30-b4c6-b4b80ceff106 --mbrtogpt -- ${jdev} + OSD_JOURNAL=$(dev_part ${jdev} ${osd_journal_partition}) + udev_settle + else + echo "The backing device ${jdev} for ${OSD_JOURNAL} does not exist on this system." + exit 1 + fi +} + +function osd_journal_prepare { + if [ -n "${OSD_JOURNAL}" ]; then + if [ -b ${OSD_JOURNAL} ]; then + OSD_JOURNAL=$(readlink -f ${OSD_JOURNAL}) + OSD_JOURNAL_PARTITION=$(echo ${OSD_JOURNAL} | sed 's/[^0-9]//g') + local jdev=$(echo ${OSD_JOURNAL} | sed 's/[0-9]//g') + if [ -z "${OSD_JOURNAL_PARTITION}" ]; then + OSD_JOURNAL=$(dev_part ${jdev} ${OSD_JOURNAL_PARTITION}) + else + OSD_JOURNAL=${OSD_JOURNAL} + fi + elif [ "x$JOURNAL_TYPE" != "xdirectory" ]; then + # The block device exists but doesn't appear to be paritioned, we will proceed with parititioning the device. + OSD_JOURNAL=$(readlink -f ${OSD_JOURNAL}) + osd_journal_create ${OSD_JOURNAL} + fi + chown ceph. ${OSD_JOURNAL} + elif [ "x$JOURNAL_TYPE" != "xdirectory" ]; then + echo "No journal device specified. OSD and journal will share ${OSD_DEVICE}" + echo "For better performance on HDD, consider moving your journal to a separate device" + fi + CLI_OPTS="${CLI_OPTS} --filestore" +} + +if ! [ "x${STORAGE_TYPE%-*}" == "xdirectory" ]; then + osd_disk_prepare +fi diff --git a/ceph-osd/templates/configmap-bin.yaml b/ceph-osd/templates/configmap-bin.yaml index 61fb26e28..b32bc9796 100644 --- a/ceph-osd/templates/configmap-bin.yaml +++ b/ceph-osd/templates/configmap-bin.yaml @@ -32,20 +32,22 @@ data: {{- end }} osd-start.sh: | {{ tuple "bin/osd/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - osd-directory.sh: | + osd-directory-ceph-disk.sh: | {{ tuple "bin/osd/_directory.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - osd-block.sh: | -{{ tuple "bin/osd/_block.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - osd-bluestore.sh: | -{{ tuple "bin/osd/_bluestore.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-block-ceph-disk.sh: | +{{ tuple "bin/osd/ceph-disk/_block.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-bluestore-ceph-disk.sh: | +{{ tuple "bin/osd/ceph-disk/_bluestore.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-init-ceph-disk.sh: | +{{ tuple "bin/osd/ceph-disk/_init-with-ceph-disk.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} osd-init.sh: | {{ tuple "bin/osd/_init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} osd-check.sh: | {{ tuple "bin/osd/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} osd-stop.sh: | {{ tuple "bin/osd/_stop.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - osd-common.sh: | -{{ tuple "bin/osd/_common.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-common-ceph-disk.sh: | +{{ tuple "bin/osd/ceph-disk/_common.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} init-dirs.sh: | {{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} helm-tests.sh: | diff --git a/ceph-osd/templates/daemonset-osd.yaml b/ceph-osd/templates/daemonset-osd.yaml index 25f9eb56c..77ad9b26d 100644 --- a/ceph-osd/templates/daemonset-osd.yaml +++ b/ceph-osd/templates/daemonset-osd.yaml @@ -89,10 +89,6 @@ spec: mountPath: /tmp/init-dirs.sh subPath: init-dirs.sh readOnly: true - - name: ceph-osd-bin - mountPath: /tmp/osd-common.sh - subPath: osd-common.sh - readOnly: true - name: ceph-osd-etc mountPath: /etc/ceph/storage.json subPath: storage.json @@ -155,6 +151,8 @@ spec: # value: directory - name: CLUSTER value: "ceph" + - name: DEPLOY_TOOL + value: {{ .Values.deploy.tool }} - name: CEPH_GET_ADMIN_KEY value: "1" - name: NAMESPACE @@ -178,8 +176,12 @@ spec: subPath: osd-init.sh readOnly: true - name: ceph-osd-bin - mountPath: /tmp/osd-common.sh - subPath: osd-common.sh + mountPath: /tmp/init-ceph-disk.sh + subPath: osd-init-ceph-disk.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/osd-common-ceph-disk.sh + subPath: osd-common-ceph-disk.sh readOnly: true - name: ceph-osd-etc mountPath: /etc/ceph/ceph.conf.template @@ -229,6 +231,8 @@ spec: # value: directory - name: CLUSTER value: "ceph" + - name: DEPLOY_TOOL + value: {{ .Values.deploy.tool }} - name: CEPH_GET_ADMIN_KEY value: "1" - name: NAMESPACE @@ -273,16 +277,16 @@ spec: subPath: osd-start.sh readOnly: true - name: ceph-osd-bin - mountPath: /tmp/osd-directory.sh - subPath: osd-directory.sh + mountPath: /tmp/osd-directory-ceph-disk.sh + subPath: osd-directory-ceph-disk.sh readOnly: true - name: ceph-osd-bin - mountPath: /tmp/osd-block.sh - subPath: osd-block.sh + mountPath: /tmp/osd-block-ceph-disk.sh + subPath: osd-block-ceph-disk.sh readOnly: true - name: ceph-osd-bin - mountPath: /tmp/osd-bluestore.sh - subPath: osd-bluestore.sh + mountPath: /tmp/osd-bluestore-ceph-disk.sh + subPath: osd-bluestore-ceph-disk.sh readOnly: true - name: ceph-osd-bin mountPath: /tmp/osd-check.sh @@ -297,8 +301,8 @@ spec: subPath: utils-checkDNS.sh readOnly: true - name: ceph-osd-bin - mountPath: /tmp/osd-common.sh - subPath: osd-common.sh + mountPath: /tmp/osd-common-ceph-disk.sh + subPath: osd-common-ceph-disk.sh readOnly: true - name: ceph-osd-bin mountPath: /tmp/utils-defragOSDs.sh diff --git a/ceph-osd/values.yaml b/ceph-osd/values.yaml index 0650d95da..d307df074 100644 --- a/ceph-osd/values.yaml +++ b/ceph-osd/values.yaml @@ -42,6 +42,9 @@ labels: node_selector_key: ceph-osd node_selector_value: enabled +deploy: + tool: "ceph-disk" + pod: security_context: osd: