diff --git a/ceph-osd/Chart.yaml b/ceph-osd/Chart.yaml index 1edf4fad2..2f3a576a5 100644 --- a/ceph-osd/Chart.yaml +++ b/ceph-osd/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph OSD name: ceph-osd -version: 0.1.15 +version: 0.1.16 home: https://github.com/ceph/ceph ... diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl index a02f71479..98979dbd2 100644 --- a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl @@ -57,6 +57,9 @@ function common_cleanup() { # Run a command within the global synchronization lock function locked() { + # Don't log every command inside locked() to keep logs cleaner + { set +x; } 2>/dev/null + local LOCK_SCOPE=0 # Allow locks to be re-entrant to avoid deadlocks @@ -66,12 +69,17 @@ function locked() { fi # Execute the synchronized command + set -x "$@" + { set +x; } 2>/dev/null # Only unlock if the lock was obtained in this scope if [[ ${LOCK_SCOPE} -ne 0 ]]; then unlock fi + + # Re-enable command logging + set -x } # Alias commands that interact with disks so they are always synchronized @@ -304,21 +312,15 @@ function zap_extra_partitions { function disk_zap { # Run all the commands that ceph-disk zap uses to clear a disk local device=${1} - local device_filter=$(basename "${device}") - local lv_name=$(get_lv_name_from_device "${device}" lv) - local dm_devices=$(get_lvm_path_from_device "pv_name=~${device_filter},lv_name=~ceph") + local dm_devices=$(get_dm_devices_from_osd_device "${device}" | xargs) for dm_device in ${dm_devices}; do - if [[ ! -z ${dm_device} ]] && [[ ! -z $(dmsetup ls | grep ${dm_device}) ]]; then + if [[ "$(dmsetup ls | grep ${dm_device})" ]]; then dmsetup remove ${dm_device} fi done - if [[ ! -z "${lv_name}" ]]; then - local logical_volumes=$(lvdisplay | grep "LV Path" | grep "${lv_name}" | awk '/ceph/{print $3}' | tr '\n' ' ') - for logical_volume in ${logical_volumes}; do - if [[ ! -z ${logical_volume} ]]; then - lvremove -y ${logical_volume} - fi - done + local logical_volumes=$(get_lv_paths_from_osd_device "${device}" | xargs) + if [[ "${logical_volumes}" ]]; then + lvremove -y ${logical_volumes} fi local volume_group=$(pvdisplay -ddd -v ${device} | grep "VG Name" | awk '/ceph/{print $3}' | grep "ceph") if [[ ${volume_group} ]]; then @@ -503,11 +505,24 @@ function get_block_uuid_from_device { get_lvm_tag_from_device ${device} ceph.block_uuid } -function get_lvm_path_from_device { - select="$1" +function get_dm_devices_from_osd_device { + device="$1" + pv_uuid=$(pvdisplay -ddd -v ${device} | awk '/PV UUID/{print $3}') - options="--noheadings -o lv_dm_path" - pvs ${options} -S "${select}" | tr -d ' ' + # Return the list of dm devices that belong to the osd + if [[ "${pv_uuid}" ]]; then + dmsetup ls | grep "$(echo "${pv_uuid}" | sed 's/-/--/g')" | awk '{print $1}' + fi +} + +function get_lv_paths_from_osd_device { + device="$1" + pv_uuid=$(pvdisplay -ddd -v ${device} | awk '/PV UUID/{print $3}') + + # Return the list of lvs that belong to the osd + if [[ "${pv_uuid}" ]]; then + lvdisplay | grep "LV Path" | grep "${pv_uuid}" | awk '{print $3}' + fi } function get_vg_name_from_device { diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl index 7daac65a7..5dacde20b 100644 --- a/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl @@ -195,28 +195,41 @@ function prep_device { vg_name=$(get_vg_name_from_device ${BLOCK_DEVICE}) lv_name=$(get_lv_name_from_device ${data_disk} ${device_type}) VG=$(vgs --noheadings -o vg_name -S "vg_name=${vg_name}" | tr -d '[:space:]') - if [[ $VG ]]; then + if [[ "${VG}" ]]; then DEVICE_OSD_ID=$(get_osd_id_from_volume "/dev/${vg_name}/${lv_name}") CEPH_LVM_PREPARE=1 - if [ -n "${OSD_ID}" ]; then - if [ "${DEVICE_OSD_ID}" == "${OSD_ID}" ]; then + if [[ -n "${DEVICE_OSD_ID}" ]] && [[ -n "${OSD_ID}" ]]; then + if [[ "${DEVICE_OSD_ID}" == "${OSD_ID}" ]]; then CEPH_LVM_PREPARE=0 else disk_zap "${OSD_DEVICE}" fi fi + logical_volumes="$(lvs --noheadings -o lv_name ${VG} | xargs)" + for volume in ${logical_volumes}; do + data_volume=$(echo ${volume} | sed -E -e 's/db|wal/lv/g') + if [[ -z $(lvs --noheadings -o lv_name -S "lv_name=${data_volume}") ]]; then + # DB or WAL volume without a corresponding data volume, remove it + lvremove -y /dev/${VG}/${volume} + fi + done else - logical_devices=$(get_lvm_path_from_device "pv_name=~${BLOCK_DEVICE},lv_name=~${lv_name}") - if [[ -n "$logical_devices" ]]; then - dmsetup remove $logical_devices - disk_zap "${OSD_DEVICE}" - CEPH_LVM_PREPARE=1 + if [[ "${vg_name}" ]]; then + logical_devices=$(get_dm_devices_from_osd_device "${data_disk}") + device_filter=$(echo "${vg_name}" | sed 's/-/--/g') + logical_devices=$(echo "${logical_devices}" | grep "${device_filter}" | xargs) + if [[ "$logical_devices" ]]; then + dmsetup remove $logical_devices + disk_zap "${OSD_DEVICE}" + CEPH_LVM_PREPARE=1 + fi fi random_uuid=$(uuidgen) vgcreate "ceph-vg-${random_uuid}" "${BLOCK_DEVICE}" VG=$(get_vg_name_from_device ${BLOCK_DEVICE}) vgrename "ceph-vg-${random_uuid}" "${VG}" fi + udev_settle logical_volume=$(lvs --noheadings -o lv_name -S "lv_name=${lv_name}" | tr -d '[:space:]') if [[ $logical_volume != "${lv_name}" ]]; then lvcreate -L "${BLOCK_DEVICE_SIZE}" -n "${lv_name}" "${VG}" @@ -295,20 +308,16 @@ function osd_disk_prepare { elif [[ $(sgdisk --print ${OSD_DEVICE} | grep "F800") ]]; then DM_DEV=${OSD_DEVICE}$(sgdisk --print ${OSD_DEVICE} | grep "F800" | awk '{print $1}') CEPH_DISK_USED=1 - elif [[ $(lsblk ${OSD_DEVICE}|grep -i ceph) ]]; then - CEPH_DISK_USED=1 else - dm_lv_name="$(get_lv_name_from_device ${OSD_DEVICE} lv | sed 's/-/--/g')" - if [[ ! -z "${dm_lv_name}" ]] && [[ ! -z "$(dmsetup ls | grep ${dm_lv_name})" ]]; then - CEPH_DISK_USED=1 - fi - if [[ ${OSD_FORCE_REPAIR} -eq 1 ]] && [ ${CEPH_DISK_USED} -ne 1 ]; then - echo "${OSD_DEVICE} isn't clean, zapping it because OSD_FORCE_REPAIR is enabled" - disk_zap ${OSD_DEVICE} - else - echo "${OSD_DEVICE} isn't clean, but OSD_FORCE_REPAIR isn't enabled." - echo "Please set OSD_FORCE_REPAIR to '1' if you want to zap this disk." - exit 1 + if [[ ${CEPH_DISK_USED} -eq 1 ]]; then + if [[ ${OSD_FORCE_REPAIR} -eq 1 ]]; then + echo "${OSD_DEVICE} isn't clean, zapping it because OSD_FORCE_REPAIR is enabled" + disk_zap ${OSD_DEVICE} + else + echo "${OSD_DEVICE} isn't clean, but OSD_FORCE_REPAIR isn't enabled." + echo "Please set OSD_FORCE_REPAIR to '1' if you want to zap this disk." + exit 1 + fi fi fi fi @@ -456,7 +465,7 @@ function osd_disk_prepare { CLI_OPTS="${CLI_OPTS} --crush-device-class ${DEVICE_CLASS}" fi - if [[ CEPH_LVM_PREPARE -eq 1 ]]; then + if [[ ${CEPH_LVM_PREPARE} -eq 1 ]]; then ceph-volume lvm -v prepare ${CLI_OPTS} udev_settle fi