[ceph-osd] Use lvm commands instead of ceph-volume to get OSD properties
This change removes "ceph-volume inventory" and "ceph-volume lvm list" commands from the ceph-volume OSD initialization script and Bluestore start script and replaces them with "pvdisplay" and "lvs" to retrieve lvm tags directly from lvm volumes instead. Ceph-volume makes repeated calls to blkid, which is very slow in some cases and deadlocks in others when there are RBDs mapped on the host. Change-Id: Ia999770d4a59729e38dbb494b34c30e5a1b36a8b
This commit is contained in:
Steve Taylor
Stephen Taylor
parent
8dbd488605
commit
a12ea0244e
@ -39,7 +39,7 @@ CEPH_OSD_OPTIONS=""
|
||||
|
||||
udev_settle
|
||||
|
||||
OSD_ID=$(ceph-volume inventory ${OSD_DEVICE} | grep "osd id" | awk '{print $3}')
|
||||
OSD_ID=$(get_osd_id_from_device ${OSD_DEVICE})
|
||||
simple_activate=0
|
||||
if [[ -z ${OSD_ID} ]]; then
|
||||
echo "Looks like ceph-disk has been used earlier to activate the OSD."
|
||||
@ -49,7 +49,7 @@ if [[ -z ${OSD_ID} ]]; then
|
||||
umount ${tmpmnt}
|
||||
simple_activate=1
|
||||
fi
|
||||
OSD_FSID=$(ceph-volume inventory ${OSD_DEVICE} | grep "osd fsid" | awk '{print $3}')
|
||||
OSD_FSID=$(get_osd_fsid_from_device ${OSD_DEVICE})
|
||||
if [[ -z ${OSD_FSID} ]]; then
|
||||
echo "Looks like ceph-disk has been used earlier to activate the OSD."
|
||||
tmpmnt=$(mktemp -d)
|
||||
@ -73,7 +73,7 @@ else
|
||||
--auto-detect-objectstore \
|
||||
--no-systemd ${OSD_ID} ${OSD_FSID}
|
||||
# Cross check the db and wal symlinks if missed
|
||||
DB_DEV=$(ceph-volume lvm list ${OSD_DEVICE} | grep "db device" | awk '{print $3}')
|
||||
DB_DEV=$(get_osd_db_device_from_device ${OSD_DEVICE})
|
||||
if [[ ! -z ${DB_DEV} ]]; then
|
||||
if [[ ! -h /var/lib/ceph/osd/ceph-${OSD_ID}/block.db ]]; then
|
||||
ln -snf ${DB_DEV} /var/lib/ceph/osd/ceph-${OSD_ID}/block.db
|
||||
@ -81,7 +81,7 @@ else
|
||||
chown -h ceph:ceph /var/lib/ceph/osd/ceph-${OSD_ID}/block.db
|
||||
fi
|
||||
fi
|
||||
WAL_DEV=$(ceph-volume lvm list ${OSD_DEVICE} | grep "wal device" | awk '{print $3}')
|
||||
WAL_DEV=$(get_osd_wal_device_from_device ${OSD_DEVICE})
|
||||
if [[ ! -z ${WAL_DEV} ]]; then
|
||||
if [[ ! -h /var/lib/ceph/osd/ceph-${OSD_ID}/block.wal ]]; then
|
||||
ln -snf ${WAL_DEV} /var/lib/ceph/osd/ceph-${OSD_ID}/block.wal
|
||||
|
||||
@ -251,3 +251,74 @@ function udev_settle {
|
||||
done
|
||||
}
|
||||
|
||||
# Helper function to get an lvm tag from a logical volume
|
||||
function get_lvm_tag_from_volume {
|
||||
logical_volume="$1"
|
||||
tag="$2"
|
||||
|
||||
if [[ -z "${logical_volume}" ]]; then
|
||||
# Return an empty string if the logical volume doesn't exist
|
||||
echo
|
||||
else
|
||||
# Get and return the specified tag from the logical volume
|
||||
echo "$(lvs -o lv_tags ${logical_volume} | tr ',' '\n' | grep ${tag} | cut -d'=' -f2)"
|
||||
fi
|
||||
}
|
||||
|
||||
function get_lvm_tag_from_device {
|
||||
device="$1"
|
||||
tag="$2"
|
||||
# Attempt to get a logical volume for the physical device
|
||||
logical_volume="$(pvdisplay -m ${device} | awk '/Logical volume/{print $3}')"
|
||||
|
||||
# Use get_lvm_tag_from_volume to get the specified tag from the logical volume
|
||||
echo "$(get_lvm_tag_from_volume ${logical_volume} ${tag})"
|
||||
}
|
||||
|
||||
# Helper function get a cluster FSID from a physical device
|
||||
function get_cluster_fsid_from_device {
|
||||
device="$1"
|
||||
|
||||
# Use get_lvm_tag_from_device to get the cluster FSID from the device
|
||||
echo "$(get_lvm_tag_from_device ${device} ceph.cluster_fsid)"
|
||||
}
|
||||
|
||||
# Helper function to get an OSD ID from a logical volume
|
||||
function get_osd_id_from_volume {
|
||||
logical_volume="$1"
|
||||
|
||||
# Use get_lvm_tag_from_volume to get the OSD ID from the logical volume
|
||||
echo "$(get_lvm_tag_from_volume ${logical_volume} ceph.osd_id)"
|
||||
}
|
||||
|
||||
# Helper function get an OSD ID from a physical device
|
||||
function get_osd_id_from_device {
|
||||
device="$1"
|
||||
|
||||
# Use get_lvm_tag_from_device to get the OSD ID from the device
|
||||
echo "$(get_lvm_tag_from_device ${device} ceph.osd_id)"
|
||||
}
|
||||
|
||||
# Helper function get an OSD FSID from a physical device
|
||||
function get_osd_fsid_from_device {
|
||||
device="$1"
|
||||
|
||||
# Use get_lvm_tag_from_device to get the OSD FSID from the device
|
||||
echo "$(get_lvm_tag_from_device ${device} ceph.osd_fsid)"
|
||||
}
|
||||
|
||||
# Helper function get an OSD DB device from a physical device
|
||||
function get_osd_db_device_from_device {
|
||||
device="$1"
|
||||
|
||||
# Use get_lvm_tag_from_device to get the OSD DB device from the device
|
||||
echo "$(get_lvm_tag_from_device ${device} ceph.db_device)"
|
||||
}
|
||||
|
||||
# Helper function get an OSD WAL device from a physical device
|
||||
function get_osd_wal_device_from_device {
|
||||
device="$1"
|
||||
|
||||
# Use get_lvm_tag_from_device to get the OSD WAL device from the device
|
||||
echo "$(get_lvm_tag_from_device ${device} ceph.wal_device)"
|
||||
}
|
||||
|
||||
@ -62,7 +62,7 @@ function osd_disk_prepare {
|
||||
CEPH_LVM_PREPARE=1
|
||||
osd_dev_string=$(echo ${OSD_DEVICE} | awk -F "/" '{print $2}{print $3}' | paste -s -d'-')
|
||||
udev_settle
|
||||
OSD_ID=$(ceph-volume inventory ${OSD_DEVICE} | grep "osd id" | awk '{print $3}')
|
||||
OSD_ID=$(get_osd_id_from_device ${OSD_DEVICE})
|
||||
if [ "${OSD_BLUESTORE:-0}" -ne 1 ]; then
|
||||
if [[ ! -z ${OSD_ID} ]]; then
|
||||
DM_NUM=$(dmsetup ls | grep $(lsblk -J ${OSD_DEVICE} | jq -r '.blockdevices[].children[].name') | awk '{print $2}' | cut -d':' -f2 | cut -d')' -f1)
|
||||
@ -197,8 +197,8 @@ function osd_disk_prepare {
|
||||
if [[ ${block_db_string} == ${block_wal_string} ]]; then
|
||||
if [[ $(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_db_string}") ]]; then
|
||||
VG=$(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_db_string}")
|
||||
WAL_OSD_ID=$(ceph-volume lvm list /dev/ceph-db-wal-${block_wal_string}/ceph-wal-${osd_dev_string} | grep "osd id" | awk '{print $3}')
|
||||
DB_OSD_ID=$(ceph-volume lvm list /dev/ceph-db-wal-${block_db_string}/ceph-db-${osd_dev_string} | grep "osd id" | awk '{print $3}')
|
||||
WAL_OSD_ID=$(get_osd_id_from_volume /dev/ceph-db-wal-${block_wal_string}/ceph-wal-${osd_dev_string})
|
||||
DB_OSD_ID=$(get_osd_id_from_volume /dev/ceph-db-wal-${block_db_string}/ceph-db-${osd_dev_string})
|
||||
if [ ! -z ${OSD_ID} ] && ([ ${WAL_OSD_ID} != ${OSD_ID} ] || [ ${DB_OSD_ID} != ${OSD_ID} ]); then
|
||||
echo "Found VG, but corresponding DB || WAL are not, zapping the ${OSD_DEVICE}"
|
||||
disk_zap ${OSD_DEVICE}
|
||||
@ -236,7 +236,7 @@ function osd_disk_prepare {
|
||||
else
|
||||
if [[ $(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_db_string}") ]]; then
|
||||
VG=$(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_db_string}")
|
||||
DB_OSD_ID=$(ceph-volume lvm list /dev/ceph-db-wal-${block_db_string}/ceph-db-${block_db_string} | grep "osd id" | awk '{print $3}')
|
||||
DB_OSD_ID=$(get_osd_id_from_volume /dev/ceph-db-wal-${block_db_string}/ceph-db-${block_db_string})
|
||||
if [ ! -z ${OSD_ID} ] && [ ${DB_OSD_ID} != ${OSD_ID} ]; then
|
||||
echo "Found VG, but corresponding DB is not, zapping the ${OSD_DEVICE}"
|
||||
disk_zap ${OSD_DEVICE}
|
||||
@ -262,7 +262,7 @@ function osd_disk_prepare {
|
||||
fi
|
||||
if [[ $(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_wal_string}") ]]; then
|
||||
VG=$(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_wal_string}")
|
||||
WAL_OSD_ID=$(ceph-volume lvm list /dev/ceph-db-wal-${block_wal_string}/ceph-wal-${block_wal_string} | grep "osd id" | awk '{print $3}')
|
||||
WAL_OSD_ID=$(get_osd_id_from_volume /dev/ceph-db-wal-${block_wal_string}/ceph-wal-${block_wal_string})
|
||||
if [ ! -z ${OSD_ID} ] && [ ${WAL_OSD_ID} != ${OSD_ID} ]; then
|
||||
echo "Found VG, but corresponding WAL is not, zapping the ${OSD_DEVICE}"
|
||||
disk_zap ${OSD_DEVICE}
|
||||
@ -298,7 +298,7 @@ function osd_disk_prepare {
|
||||
elif [[ -z ${BLOCK_DB} && ${BLOCK_WAL} ]]; then
|
||||
if [[ $(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_wal_string}") ]]; then
|
||||
VG=$(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_wal_string}")
|
||||
WAL_OSD_ID=$(ceph-volume lvm list /dev/ceph-wal-${block_wal_string}/ceph-wal-${osd_dev_string} | grep "osd id" | awk '{print $3}')
|
||||
WAL_OSD_ID=$(get_osd_id_from_volume /dev/ceph-wal-${block_wal_string}/ceph-wal-${osd_dev_string})
|
||||
if [ ! -z ${OSD_ID} ] && [ ${WAL_OSD_ID} != ${OSD_ID} ]; then
|
||||
echo "Found VG, but corresponding WAL is not, zapping the ${OSD_DEVICE}"
|
||||
disk_zap ${OSD_DEVICE}
|
||||
@ -329,7 +329,7 @@ function osd_disk_prepare {
|
||||
elif [[ ${BLOCK_DB} && -z ${BLOCK_WAL} ]]; then
|
||||
if [[ $(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_db_string}") ]]; then
|
||||
VG=$(vgdisplay | grep "VG Name" | awk '{print $3}' | grep "${block_db_string}")
|
||||
DB_OSD_ID=$(ceph-volume lvm list /dev/ceph-db-${block_db_string}/ceph-db-${osd_dev_string} | grep "osd id" | awk '{print $3}')
|
||||
DB_OSD_ID=$(get_osd_id_from_volume /dev/ceph-db-${block_db_string}/ceph-db-${osd_dev_string})
|
||||
if [ ! -z ${OSD_ID} ] && [ ${DB_OSD_ID} != ${OSD_ID} ]; then
|
||||
echo "Found VG, but corresponding DB is not, zapping the ${OSD_DEVICE}"
|
||||
disk_zap ${OSD_DEVICE}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user