From 669c9d418af3ade5f56a7ef5e012c5497aa674ba Mon Sep 17 00:00:00 2001 From: Philipp Marek Date: Wed, 11 Nov 2015 11:07:37 +0100 Subject: [PATCH] Using drbdmanage in multi-node setups. This should basically be useful standalone (on bare-bones) nodes, too, to add storage nodes to an OpenStack cluster. Furthermore, try to get rid of some "long line" warnings. Change-Id: I9e6d4131928e81262d6111b92f757ffc0ccc52d2 --- README.rst | 16 ++++++- devstack/lib/drbd_devstack | 87 ++++++++++++++++++++++++++++++-------- devstack/override-defaults | 2 +- devstack/plugin.sh | 11 +++++ devstack/settings | 4 +- 5 files changed, 98 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index 71b83d5..0d56566 100644 --- a/README.rst +++ b/README.rst @@ -12,4 +12,18 @@ This plugin helps to install the DRBD backend for Cinder Usage ----- -???? +Put some lines similar to these in your `local.conf`: + + enable_plugin drbd-devstack git://git.openstack.org/stackforge/drbd-devstack + enable_service drbd-devstack + CINDER_ENABLED_BACKENDS=drbd:drbdmanage + + +A few environment variables can be set to modify the behaviour: + + * `CINDER_DRBD_NO_STORAGE` means `--no-storage` + * `CINDER_DRBD_NO_CV` gets translated to `--no-control-volume`. + This is allowed only for additional nodes, and not the initial drbdmanage node that *has* to create a control volume. + + +Secondary nodes are detected by having `$SERVICE_HOST` unequal to `$HOST_IP`, and therefore register themselves as additional nodes in drbdmanage. diff --git a/devstack/lib/drbd_devstack b/devstack/lib/drbd_devstack index 00423e4..945ca4b 100755 --- a/devstack/lib/drbd_devstack +++ b/devstack/lib/drbd_devstack @@ -1,6 +1,9 @@ #!/bin/bash # vim: set et ts=4 sw=4 ft=sh : +# +DRBDMANAGE_DBUS_AUTH_FILE=/etc/dbus-1/system.d/org.drbd.drbdmanaged-stack.conf +DRBDBASEURL=http://openstack-ci-pkgs.linbit.com/packages/003/trusty function pre_install_drbd_devstack { # Install OS packages, if necessary @@ -9,32 +12,35 @@ function pre_install_drbd_devstack { fi packages=( + drbd-utils_8.9.4-1_amd64.deb drbd8-utils_8.9.4-1_amd64.deb drbd-dkms_9.0.0-2_all.deb - drbd-utils_8.9.4-1_amd64.deb - python-drbdmanage_0.50-1.1_all.deb + python-drbdmanage_0.50-1.2_all.deb ) + # get packages for p in "${packages[@]}"; do if [[ ! -f "${FILES}/${p}" ]]; then # If there are newer packages, change the download number here. - wget "http://openstack-ci-pkgs.linbit.com/packages/003/trusty/$p" -O "${FILES}/${p}" + wget "$DRBDBASEURL/$p" -O "${FILES}/${p}" fi done + # install packages + sudo apt-get update + sudo apt-get install --yes debhelper python-dbus dbus \ + lvm2 thin-provisioning-tools for i in "${packages[@]}"; do if [[ -f "${FILES}/${i}" ]]; then echo "installing ${i}" - sudo dpkg -i "${FILES}/${i}" || true + sudo dpkg --force-confnew --install "${FILES}/${i}" || true fi done # now go fetch :) echo echo "installing deps" - #sudo apt-get update - sudo apt-get install -f -y - sudo apt-get install python-dbus -y + sudo apt-get install --fix-broken --yes return 0 } @@ -43,7 +49,8 @@ function install_drbd_devstack { # Write DRBDmanage configuration; # use the single-thinpool driver for these tests. - sudo sed -i "s/^drbdctrl-vg\s*=.*/drbdctrl-vg = ${DRBD_DRBDCTRL_VG}/g" /etc/drbdmanaged.cfg + sudo sed -i "s/^drbdctrl-vg\s*=.*/drbdctrl-vg = ${DRBD_DRBDCTRL_VG}/g" \ + /etc/drbdmanaged.cfg echo " [LOCAL] storage-plugin = drbdmanage.storage.lvm_thinlv.LvmThinLv @@ -55,7 +62,7 @@ pool-name = drbdthinpool " | sudo tee /etc/drbdmanaged.cfg # allow the stack user access to drbdmanage - sudo tee /etc/dbus-1/system.d/org.drbd.drbdmanaged-stack.conf > /dev/null << "EOF" + sudo tee "$DRBDMANAGE_DBUS_AUTH_FILE" > /dev/null << "EOF" @@ -69,6 +76,10 @@ EOF # done. } +function _find_lo_for_dev { + sudo losetup -a | grep "$1" +} + function _drbd_make_vg { local vg_name="${1:?No VG name given}" local vg_size="$2" @@ -93,7 +104,7 @@ function _drbd_make_vg { fi # if the file is already assigned a loop device, don't reassign - if ! sudo losetup -a | grep "${vg_lo_dev}" | grep -q "${vg_name}" ; then + if ! _find_lo_for_dev "${vg_lo_dev}" | grep -q "${vg_name}" ; then sudo losetup "${vg_dev}" "${FILES}/${vg_name}" fi @@ -133,6 +144,12 @@ function _modify_udev_file { fi } +function _this_is_the_initial_node { + [[ -z "$SERVICE_HOST" || + "$SERVICE_HOST" == "127.0.0.1" || + "$SERVICE_HOST" == "$HOST_IP" ]] +} + function configure_drbd_devstack { # Configure the service. # This gets called before starting the c-vol service; the next callback, @@ -143,12 +160,18 @@ function configure_drbd_devstack { # Initialize and start the service. # need to setup loopback device(s) for DRBD - _drbd_make_vg "${DRBD_DRBDCTRL_VG:?DRBD_DRBDCTRL_VG is not set}" "${DRBD_DRBDCTRL_VG_SZ}" "${DRBD_DRBDCTRL_LODEV}" + _drbd_make_vg "${DRBD_DRBDCTRL_VG:?DRBD_DRBDCTRL_VG is not set}" \ + "${DRBD_DRBDCTRL_VG_SZ}" "${DRBD_DRBDCTRL_LODEV}" # Do the same thing for the DATA volume group - _drbd_make_vg "${DRBD_DATA_VG:?DRBD_DATA_VG is not set}" "${DRBD_DATA_VG_SZ}" "${DRBD_DATA_LODEV}" + _drbd_make_vg "${DRBD_DATA_VG:?DRBD_DATA_VG is not set}" \ + "${DRBD_DATA_VG_SZ}" "${DRBD_DATA_LODEV}" + + local vg_size=$(LC_ALL=C sudo vgdisplay --columns --units M \ + --noheadings -o vg_free --nosuffix "${DRBD_DATA_VG}") + # No quotes, so that the whitespace around the number gets eaten by the shell + local thinpool_size=$( echo $vg_size \* 30 / 32 - 64 | bc ) - local thinpool_size=$(echo $(LC_ALL=C sudo vgdisplay --columns --units M --noheadings -o vg_free --nosuffix "${DRBD_DATA_VG}") \* 30 / 32 - 64 | bc) if ! sudo lvdisplay "${DRBD_DATA_VG}/drbdthinpool" ; then sudo /sbin/lvcreate -L "${thinpool_size}"M -T "${DRBD_DATA_VG}/drbdthinpool" fi @@ -175,15 +198,43 @@ function configure_drbd_devstack { 'ENV{DM_NAME}=="'"${DRBD_DRBDCTRL_VG}"'-.drbdctrl' \ 's:SUBSYSTEM.="block",.*GOTO="\(.*\)":&\nENV{DM_NAME}=="'"${DRBD_DRBDCTRL_VG}"'-.drbdctrl", GOTO="\1"\n:' - # initialize drbdmanage - sudo drbdmanage init --quiet - sudo drbdmanage shutdown --quiet + # drbdmanage modii: --no-control-volume --no-storage --no-autojoin + + no_stor="${CINDER_DRBD_NO_STORAGE:+--no-storage}" + + # initialize drbdmanage + if _this_is_the_initial_node ; then + # No quotes, so that it can expand to an empty string (and get ignored), too. + sudo drbdmanage init --quiet $no_stor + else + # Do we want $HOST_IP, or a possibly different one that should be reachable by the SERVICE_HOST? + # Is the $SERVICE_HOST always a drbdmanage node, or should we go to $CINDER_SERVICE_HOST? + # for now some debug output + set | grep SERVICE_HOST + + local my_address="$HOST_IP" + if [[ -z "$my_address" ]] ; then + # Fallback, in case that script gets called standalone. + my_address="$(ip -oneline route get "$SERVICE_HOST" | \ + grep "$SERVICE_HOST" | awk '/src (.*)/ { print $5 }')" + fi + echo "I am from $my_address" # ;) + + no_cv="${CINDER_DRBD_NO_CV:+--no-control-volume}" + + ssh -oBatchMode=yes -oStrictHostKeyChecking=no stack@"$SERVICE_HOST" \ + sudo drbdmanage new-node --no-autojoin "$no_cv" "$no_stor" \ + --quiet "$HOSTNAME" "$my_address" + + ssh -oBatchMode=yes -oStrictHostKeyChecking=no stack@"$SERVICE_HOST" \ + sudo drbdmanage howto-join "$HOSTNAME" --quiet | sudo bash + fi + + sudo drbdmanage shutdown --quiet sudo drbdmanage debug 'set loglevel=debug' sudo drbdmanage nodes - # FIXME: multi-node setups - iniset $CINDER_CONF "$be_name" volume_backend_name "$be_name" iniset $CINDER_CONF "$be_name" volume_driver cinder.volume.drivers.drbdmanagedrv.DrbdManageDriver diff --git a/devstack/override-defaults b/devstack/override-defaults index 200211c..8afd461 100644 --- a/devstack/override-defaults +++ b/devstack/override-defaults @@ -1,4 +1,4 @@ # Plug-in overrides -CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-drbd:drbd} +CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-drbd:drbdmanage} diff --git a/devstack/plugin.sh b/devstack/plugin.sh index d7b8316..8bc8093 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -42,4 +42,15 @@ if is_service_enabled drbd-devstack; then source "$dir/devstack/lib/drbd_devstack" cleanup_drbd_devstack fi + + if [[ "$1" == "make-a-storage-node" ]]; then + # Remove state and transient data + # Remember clean.sh first calls unstack.sh + source "$dir/devstack/settings" + source "$dir/devstack/lib/drbd_devstack" + pre_install_drbd_devstack + install_drbd_devstack + configure_drbd_devstack + init_drbd_devstack + fi fi diff --git a/devstack/settings b/devstack/settings index 13e3ca8..48daee1 100644 --- a/devstack/settings +++ b/devstack/settings @@ -1,10 +1,10 @@ # Devstack settings -FILES="${FILES:-$TOP_DIR/files}" +FILES="${FILES:-${TOP_DIR:-/tmp}/files}" TEMPEST_STORAGE_PROTOCOL=iSCSI DRBD_DRBDCTRL_VG_SZ=1G DRBD_DRBDCTRL_LODEV=loop66 DRBD_DRBDCTRL_VG=drbdpool -DRBD_DATA_VG_SZ="$VOLUME_BACKING_FILE_SIZE" +DRBD_DATA_VG_SZ="${VOLUME_BACKING_FILE_SIZE:-14000M}" DRBD_DATA_VG=drbddata DRBD_DATA_LODEV=loop67