From 285959c8633437b89060bdbeff84c3a2ad1b083c Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Mon, 21 Aug 2017 22:02:44 -0500 Subject: [PATCH] Disk targeting This PS adds a disc targeting framework for OSH Components for services like Ceph, Swift, and Cinder (LVM) to use when targeting phsical discs. It uses the bus ID of the attached device to simplfy management of large quanitites of physical hardware. Change-Id: I1cb227e43ed5394d4c68c6156047889574a0e869 --- tools/gate/funcs/common.sh | 115 ++++++++++++++++++++------ tools/gate/funcs/kube.sh | 12 ++- tools/gate/setup_gate.sh | 6 +- tools/gate/setup_gate_worker_nodes.sh | 1 - tools/gate/vars.sh | 8 +- 5 files changed, 110 insertions(+), 32 deletions(-) diff --git a/tools/gate/funcs/common.sh b/tools/gate/funcs/common.sh index 3639ec575c..37f3794773 100644 --- a/tools/gate/funcs/common.sh +++ b/tools/gate/funcs/common.sh @@ -34,7 +34,9 @@ function base_install { iptables \ ipcalc \ nmap \ - lshw + lshw \ + jq \ + python-pip elif [ "x$HOST_OS" == "xcentos" ]; then sudo yum install -y \ epel-release @@ -44,16 +46,26 @@ function base_install { iptables \ initscripts \ nmap \ - lshw + lshw \ + python-pip + # We need JQ 1.5 which is not currently in the CentOS or EPEL repos + sudo curl -L -o /usr/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 + sudo chmod +x /usr/bin/jq elif [ "x$HOST_OS" == "xfedora" ]; then sudo dnf install -y \ iproute \ iptables \ ipcalc \ nmap \ - lshw + lshw \ + jq \ + python2-pip fi + sudo -H pip install --upgrade pip + sudo -H pip install --upgrade setuptools + sudo -H pip install pyyaml + if [ "x$SDN_PLUGIN" == "xlinuxbridge" ]; then sdn_lb_support_install fi @@ -69,7 +81,13 @@ function base_install { fi } +function json_to_yaml { + python -c 'import sys, yaml, json; yaml.safe_dump(json.load(sys.stdin), sys.stdout, default_flow_style=False)' +} +function yaml_to_json { + python -c 'import sys, yaml, json; json.dump(yaml.safe_load(sys.stdin), sys.stdout)' +} function loopback_support_install { if [ "x$HOST_OS" == "xubuntu" ]; then @@ -93,34 +111,81 @@ function loopback_support_install { } function loopback_setup { + ORIGINAL_DISCS=$(mktemp --suffix=.txt) + ALL_DISCS=$(mktemp --suffix=.txt) + NEW_DISCS=$(mktemp --directory) + sudo rm -rf ${LOOPBACK_DIR} || true sudo mkdir -p ${LOOPBACK_DIR} - for ((LOOPBACK_DEV=1;LOOPBACK_DEV<=${LOOPBACK_DEVS};LOOPBACK_DEV++)); do - if [ "x$HOST_OS" == "xubuntu" ]; then - sudo targetcli backstores/fileio create loopback-${LOOPBACK_DEV} ${LOOPBACK_DIR}/fileio-${LOOPBACK_DEV} ${LOOPBACK_SIZE} + + COUNT=0 + IFS=','; for LOOPBACK_NAME in ${LOOPBACK_NAMES}; do + sudo lshw -class disk > ${ORIGINAL_DISCS} + IFS=' ' + let COUNT=COUNT+1 + LOOPBACK_DEVS=$(echo ${LOOPBACK_DEV_COUNT} | awk -F ',' "{ print \$${COUNT}}") + LOOPBACK_SIZE=$(echo ${LOOPBACK_SIZES} | awk -F ',' "{ print \$${COUNT}}") + for ((LOOPBACK_DEV=1;LOOPBACK_DEV<=${LOOPBACK_DEVS};LOOPBACK_DEV++)); do + if [ "x$HOST_OS" == "xubuntu" ]; then + sudo targetcli backstores/fileio create loopback-${LOOPBACK_NAME}-${LOOPBACK_DEV} ${LOOPBACK_DIR}/fileio-${LOOPBACK_NAME}-${LOOPBACK_DEV} ${LOOPBACK_SIZE} + else + sudo targetcli backstores/fileio create loopback-${LOOPBACK_NAME}-${LOOPBACK_DEV} ${LOOPBACK_DIR}/fileio-${LOOPBACK_NAME}-${LOOPBACK_DEV} ${LOOPBACK_SIZE} write_back=false + fi + done + sudo targetcli iscsi/ create iqn.2016-01.com.example:${LOOPBACK_NAME} + if ! [ "x$HOST_OS" == "xubuntu" ]; then + sudo targetcli iscsi/iqn.2016-01.com.example:${LOOPBACK_NAME}/tpg1/portals delete 0.0.0.0 3260 || true + sudo targetcli iscsi/iqn.2016-01.com.example:${LOOPBACK_NAME}/tpg1/portals create 127.0.0.1 3260 else - sudo targetcli backstores/fileio create loopback-${LOOPBACK_DEV} ${LOOPBACK_DIR}/fileio-${LOOPBACK_DEV} ${LOOPBACK_SIZE} write_back=false + #NOTE (Portdirect): Frustratingly it appears that Ubuntu's targetcli wont + # let you bind to localhost. + sudo targetcli iscsi/iqn.2016-01.com.example:${LOOPBACK_NAME}/tpg1/portals create 0.0.0.0 3260 fi + for ((LOOPBACK_DEV=1;LOOPBACK_DEV<=${LOOPBACK_DEVS};LOOPBACK_DEV++)); do + sudo targetcli iscsi/iqn.2016-01.com.example:${LOOPBACK_NAME}/tpg1/luns/ create /backstores/fileio/loopback-${LOOPBACK_NAME}-${LOOPBACK_DEV} + done + sudo targetcli iscsi/iqn.2016-01.com.example:${LOOPBACK_NAME}/tpg1/acls/ create $(sudo cat /etc/iscsi/initiatorname.iscsi | awk -F '=' '/^InitiatorName/ { print $NF}') + if [ "x$HOST_OS" == "xubuntu" ]; then + sudo targetcli iscsi/iqn.2016-01.com.example:${LOOPBACK_NAME}/tpg1 set attribute authentication=0 + fi + sudo iscsiadm --mode discovery --type sendtargets --portal 127.0.0.1 3260 + sudo iscsiadm -m node -T iqn.2016-01.com.example:${LOOPBACK_NAME} -p 127.0.0.1:3260 -l + + sudo lshw -class disk > ${ALL_DISCS} + # NOTE (Portdirect): Ugly subshell hack to suppress diff's exit code + (diff --changed-group-format="%>" --unchanged-group-format="" ${ORIGINAL_DISCS} ${ALL_DISCS} > ${NEW_DISCS}/${LOOPBACK_NAME}.raw) || true + jq -n -c -M \ + --arg devclass "${LOOPBACK_NAME}" \ + --arg device "$(awk '/bus info:/ { print $NF }' ${NEW_DISCS}/${LOOPBACK_NAME}.raw)" \ + '{($devclass): ($device|split("\n"))}' > ${NEW_DISCS}/${LOOPBACK_NAME} + rm -f ${NEW_DISCS}/${LOOPBACK_NAME}.raw done - sudo targetcli iscsi/ create iqn.2016-01.com.example:target - if ! [ "x$HOST_OS" == "xubuntu" ]; then - sudo targetcli iscsi/iqn.2016-01.com.example:target/tpg1/portals delete 0.0.0.0 3260 - sudo targetcli iscsi/iqn.2016-01.com.example:target/tpg1/portals create 127.0.0.1 3260 - else - #NOTE (Portdirect): Frustratingly it appears that Ubuntu's targetcli wont - # let you bind to localhost. - sudo targetcli iscsi/iqn.2016-01.com.example:target/tpg1/portals create 0.0.0.0 3260 + unset IFS + jq -c -s add ${NEW_DISCS}/* | jq --arg hostname "$(hostname)" -s -M '{block_devices:{($hostname):.[]}}' > ${LOOPBACK_LOCAL_DISC_INFO} + cat ${LOOPBACK_LOCAL_DISC_INFO} +} + +function loopback_dev_info_collect { + DEV_INFO_DIR=$(mktemp --dir) + cat ${LOOPBACK_LOCAL_DISC_INFO} > ${DEV_INFO_DIR}/$(hostname) + + if [ "x$INTEGRATION" == "xmulti" ]; then + for SUB_NODE in $SUB_NODE_IPS ; do + ssh-keyscan "${SUB_NODE}" >> ~/.ssh/known_hosts + ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${SUB_NODE} cat ${LOOPBACK_LOCAL_DISC_INFO} > ${DEV_INFO_DIR}/${SUB_NODE} + done fi - for ((LOOPBACK_DEV=1;LOOPBACK_DEV<=${LOOPBACK_DEVS};LOOPBACK_DEV++)); do - sudo targetcli iscsi/iqn.2016-01.com.example:target/tpg1/luns/ create /backstores/fileio/loopback-${LOOPBACK_DEV} + + touch ${LOOPBACK_DEV_INFO} + JQ_OPT='.[0]' + COUNT=1 + let ITERATIONS=$(ls -1q $DEV_INFO_DIR | wc -l) + while [ $COUNT -lt "$ITERATIONS" ]; do + JQ_OPT="$JQ_OPT * .[$COUNT]" + COUNT=$[$COUNT+1] done - sudo targetcli iscsi/iqn.2016-01.com.example:target/tpg1/acls/ create $(sudo cat /etc/iscsi/initiatorname.iscsi | awk -F '=' '/^InitiatorName/ { print $NF}') - if [ "x$HOST_OS" == "xubuntu" ]; then - sudo targetcli iscsi/iqn.2016-01.com.example:target/tpg1 set attribute authentication=0 - fi - sudo iscsiadm --mode discovery --type sendtargets --portal 127.0.0.1 - sudo iscsiadm -m node -T iqn.2016-01.com.example:target -p 127.0.0.1:3260 -l - # Display disks - sudo lshw -class disk + + (cd $DEV_INFO_DIR; jq -s "$JQ_OPT" *) | json_to_yaml >> ${LOOPBACK_DEV_INFO} + cat ${LOOPBACK_DEV_INFO} } function ceph_support_install { diff --git a/tools/gate/funcs/kube.sh b/tools/gate/funcs/kube.sh index 7fe0fde878..0a9f11ebd2 100755 --- a/tools/gate/funcs/kube.sh +++ b/tools/gate/funcs/kube.sh @@ -143,7 +143,13 @@ function kubeadm_aio_clean { /var/lib/nfs-provisioner || true } -function ceph_kube_controller_manager_replace { - sudo docker pull ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE} - sudo docker tag ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE} ${BASE_KUBE_CONTROLLER_MANAGER_IMAGE} +function kube_label_node_block_devs { + for HOST in $(cat $LOOPBACK_DEV_INFO | yaml_to_json | jq -r ".block_devices | keys? | .[]"); do + for DEV_TYPE in $(cat $LOOPBACK_DEV_INFO | yaml_to_json | jq -r ".block_devices.\"$HOST\" | keys? | .[]"); do + DEV_ADDRS=$(cat $LOOPBACK_DEV_INFO | yaml_to_json | jq -r ".block_devices.\"$HOST\".\"$DEV_TYPE\" | .[]") + for DEV_ADDR in $(cat $LOOPBACK_DEV_INFO | yaml_to_json | jq -r ".block_devices.\"$HOST\".\"$DEV_TYPE\" | .[]"); do + kubectl label node $HOST device-$DEV_TYPE-$(echo $DEV_ADDR | tr '@' '_' | tr ':' '-' )=enabled + done + done + done } diff --git a/tools/gate/setup_gate.sh b/tools/gate/setup_gate.sh index 00fbabf4ed..399f67ca52 100755 --- a/tools/gate/setup_gate.sh +++ b/tools/gate/setup_gate.sh @@ -17,6 +17,7 @@ source ${WORK_DIR}/tools/gate/vars.sh source ${WORK_DIR}/tools/gate/funcs/common.sh source ${WORK_DIR}/tools/gate/funcs/network.sh source ${WORK_DIR}/tools/gate/funcs/helm.sh +source ${WORK_DIR}/tools/gate/funcs/kube.sh # Setup the logging location: by default use the working dir as the root. rm -rf ${LOGS_DIR} || true @@ -59,7 +60,10 @@ else bash ${WORK_DIR}/tools/gate/kubeadm_aio.sh bash ${WORK_DIR}/tools/gate/setup_gate_worker_nodes.sh fi - + if [ "x$LOOPBACK_CREATE" == "xtrue" ]; then + loopback_dev_info_collect + kube_label_node_block_devs + fi # Deploy OpenStack-Helm if ! [ "x$INTEGRATION_TYPE" == "x" ]; then bash ${WORK_DIR}/tools/gate/helm_dry_run.sh diff --git a/tools/gate/setup_gate_worker_nodes.sh b/tools/gate/setup_gate_worker_nodes.sh index 3788e3bba9..59c846550d 100755 --- a/tools/gate/setup_gate_worker_nodes.sh +++ b/tools/gate/setup_gate_worker_nodes.sh @@ -14,7 +14,6 @@ set -ex : ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"} source ${WORK_DIR}/tools/gate/vars.sh -export SUB_NODE_COUNT="$(($(echo ${SUB_NODE_IPS} | wc -w) + 1))" sudo chown $(whoami) ${SSH_PRIVATE_KEY} sudo chmod 600 ${SSH_PRIVATE_KEY} diff --git a/tools/gate/vars.sh b/tools/gate/vars.sh index a0d31bbc89..c80092f89c 100755 --- a/tools/gate/vars.sh +++ b/tools/gate/vars.sh @@ -62,14 +62,18 @@ export SERVICE_TEST_TIMEOUT=${SERVICE_TEST_TIMEOUT:="600"} # Setup Loopback device options export LOOPBACK_CREATE=${LOOPBACK_CREATE:="false"} -export LOOPBACK_DEVS=${LOOPBACK_DEVS:="3"} -export LOOPBACK_SIZE=${LOOPBACK_SIZE:="500M"} +export LOOPBACK_DEV_COUNT=${LOOPBACK_DEV_COUNT:="3,3,3"} +export LOOPBACK_SIZES=${LOOPBACK_SIZES:="8192M,1024M,1024M"} +export LOOPBACK_NAMES=${LOOPBACK_NAMES:="cephosd,cephjournal,swift"} export LOOPBACK_DIR=${LOOPBACK_DIR:="/var/lib/iscsi-loopback"} +export LOOPBACK_LOCAL_DISC_INFO=${LOOPBACK_LOCAL_DISC_INFO:="/tmp/loopback-local-disc-info"} +export LOOPBACK_DEV_INFO=${LOOPBACK_DEV_INFO:="/tmp/loopback-dev-info"} # Setup Multinode params export SSH_PRIVATE_KEY=${SSH_PRIVATE_KEY:="/etc/nodepool/id_rsa"} export PRIMARY_NODE_IP=${PRIMARY_NODE_IP:="$(cat /etc/nodepool/primary_node | tail -1)"} export SUB_NODE_IPS=${SUB_NODE_IPS:="$(cat /etc/nodepool/sub_nodes)"} +export SUB_NODE_COUNT="$(($(echo ${SUB_NODE_IPS} | wc -w) + 1))" # Define OpenStack Test Params export OSH_BR_EX_ADDR=${OSH_BR_EX_ADDR:="172.24.4.1/24"}