diff --git a/tools/gate/funcs/common.sh b/tools/gate/funcs/common.sh index 3639ec575c..37f3794773 100644 --- a/tools/gate/funcs/common.sh +++ b/tools/gate/funcs/common.sh @@ -34,7 +34,9 @@ function base_install { iptables \ ipcalc \ nmap \ - lshw + lshw \ + jq \ + python-pip elif [ "x$HOST_OS" == "xcentos" ]; then sudo yum install -y \ epel-release @@ -44,16 +46,26 @@ function base_install { iptables \ initscripts \ nmap \ - lshw + lshw \ + python-pip + # We need JQ 1.5 which is not currently in the CentOS or EPEL repos + sudo curl -L -o /usr/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 + sudo chmod +x /usr/bin/jq elif [ "x$HOST_OS" == "xfedora" ]; then sudo dnf install -y \ iproute \ iptables \ ipcalc \ nmap \ - lshw + lshw \ + jq \ + python2-pip fi + sudo -H pip install --upgrade pip + sudo -H pip install --upgrade setuptools + sudo -H pip install pyyaml + if [ "x$SDN_PLUGIN" == "xlinuxbridge" ]; then sdn_lb_support_install fi @@ -69,7 +81,13 @@ function base_install { fi } +function json_to_yaml { + python -c 'import sys, yaml, json; yaml.safe_dump(json.load(sys.stdin), sys.stdout, default_flow_style=False)' +} +function yaml_to_json { + python -c 'import sys, yaml, json; json.dump(yaml.safe_load(sys.stdin), sys.stdout)' +} function loopback_support_install { if [ "x$HOST_OS" == "xubuntu" ]; then @@ -93,34 +111,81 @@ function loopback_support_install { } function loopback_setup { + ORIGINAL_DISCS=$(mktemp --suffix=.txt) + ALL_DISCS=$(mktemp --suffix=.txt) + NEW_DISCS=$(mktemp --directory) + sudo rm -rf ${LOOPBACK_DIR} || true sudo mkdir -p ${LOOPBACK_DIR} - for ((LOOPBACK_DEV=1;LOOPBACK_DEV<=${LOOPBACK_DEVS};LOOPBACK_DEV++)); do - if [ "x$HOST_OS" == "xubuntu" ]; then - sudo targetcli backstores/fileio create loopback-${LOOPBACK_DEV} ${LOOPBACK_DIR}/fileio-${LOOPBACK_DEV} ${LOOPBACK_SIZE} + + COUNT=0 + IFS=','; for LOOPBACK_NAME in ${LOOPBACK_NAMES}; do + sudo lshw -class disk > ${ORIGINAL_DISCS} + IFS=' ' + let COUNT=COUNT+1 + LOOPBACK_DEVS=$(echo ${LOOPBACK_DEV_COUNT} | awk -F ',' "{ print \$${COUNT}}") + LOOPBACK_SIZE=$(echo ${LOOPBACK_SIZES} | awk -F ',' "{ print \$${COUNT}}") + for ((LOOPBACK_DEV=1;LOOPBACK_DEV<=${LOOPBACK_DEVS};LOOPBACK_DEV++)); do + if [ "x$HOST_OS" == "xubuntu" ]; then + sudo targetcli backstores/fileio create loopback-${LOOPBACK_NAME}-${LOOPBACK_DEV} ${LOOPBACK_DIR}/fileio-${LOOPBACK_NAME}-${LOOPBACK_DEV} ${LOOPBACK_SIZE} + else + sudo targetcli backstores/fileio create loopback-${LOOPBACK_NAME}-${LOOPBACK_DEV} ${LOOPBACK_DIR}/fileio-${LOOPBACK_NAME}-${LOOPBACK_DEV} ${LOOPBACK_SIZE} write_back=false + fi + done + sudo targetcli iscsi/ create iqn.2016-01.com.example:${LOOPBACK_NAME} + if ! [ "x$HOST_OS" == "xubuntu" ]; then + sudo targetcli iscsi/iqn.2016-01.com.example:${LOOPBACK_NAME}/tpg1/portals delete 0.0.0.0 3260 || true + sudo targetcli iscsi/iqn.2016-01.com.example:${LOOPBACK_NAME}/tpg1/portals create 127.0.0.1 3260 else - sudo targetcli backstores/fileio create loopback-${LOOPBACK_DEV} ${LOOPBACK_DIR}/fileio-${LOOPBACK_DEV} ${LOOPBACK_SIZE} write_back=false + #NOTE (Portdirect): Frustratingly it appears that Ubuntu's targetcli wont + # let you bind to localhost. + sudo targetcli iscsi/iqn.2016-01.com.example:${LOOPBACK_NAME}/tpg1/portals create 0.0.0.0 3260 fi + for ((LOOPBACK_DEV=1;LOOPBACK_DEV<=${LOOPBACK_DEVS};LOOPBACK_DEV++)); do + sudo targetcli iscsi/iqn.2016-01.com.example:${LOOPBACK_NAME}/tpg1/luns/ create /backstores/fileio/loopback-${LOOPBACK_NAME}-${LOOPBACK_DEV} + done + sudo targetcli iscsi/iqn.2016-01.com.example:${LOOPBACK_NAME}/tpg1/acls/ create $(sudo cat /etc/iscsi/initiatorname.iscsi | awk -F '=' '/^InitiatorName/ { print $NF}') + if [ "x$HOST_OS" == "xubuntu" ]; then + sudo targetcli iscsi/iqn.2016-01.com.example:${LOOPBACK_NAME}/tpg1 set attribute authentication=0 + fi + sudo iscsiadm --mode discovery --type sendtargets --portal 127.0.0.1 3260 + sudo iscsiadm -m node -T iqn.2016-01.com.example:${LOOPBACK_NAME} -p 127.0.0.1:3260 -l + + sudo lshw -class disk > ${ALL_DISCS} + # NOTE (Portdirect): Ugly subshell hack to suppress diff's exit code + (diff --changed-group-format="%>" --unchanged-group-format="" ${ORIGINAL_DISCS} ${ALL_DISCS} > ${NEW_DISCS}/${LOOPBACK_NAME}.raw) || true + jq -n -c -M \ + --arg devclass "${LOOPBACK_NAME}" \ + --arg device "$(awk '/bus info:/ { print $NF }' ${NEW_DISCS}/${LOOPBACK_NAME}.raw)" \ + '{($devclass): ($device|split("\n"))}' > ${NEW_DISCS}/${LOOPBACK_NAME} + rm -f ${NEW_DISCS}/${LOOPBACK_NAME}.raw done - sudo targetcli iscsi/ create iqn.2016-01.com.example:target - if ! [ "x$HOST_OS" == "xubuntu" ]; then - sudo targetcli iscsi/iqn.2016-01.com.example:target/tpg1/portals delete 0.0.0.0 3260 - sudo targetcli iscsi/iqn.2016-01.com.example:target/tpg1/portals create 127.0.0.1 3260 - else - #NOTE (Portdirect): Frustratingly it appears that Ubuntu's targetcli wont - # let you bind to localhost. - sudo targetcli iscsi/iqn.2016-01.com.example:target/tpg1/portals create 0.0.0.0 3260 + unset IFS + jq -c -s add ${NEW_DISCS}/* | jq --arg hostname "$(hostname)" -s -M '{block_devices:{($hostname):.[]}}' > ${LOOPBACK_LOCAL_DISC_INFO} + cat ${LOOPBACK_LOCAL_DISC_INFO} +} + +function loopback_dev_info_collect { + DEV_INFO_DIR=$(mktemp --dir) + cat ${LOOPBACK_LOCAL_DISC_INFO} > ${DEV_INFO_DIR}/$(hostname) + + if [ "x$INTEGRATION" == "xmulti" ]; then + for SUB_NODE in $SUB_NODE_IPS ; do + ssh-keyscan "${SUB_NODE}" >> ~/.ssh/known_hosts + ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${SUB_NODE} cat ${LOOPBACK_LOCAL_DISC_INFO} > ${DEV_INFO_DIR}/${SUB_NODE} + done fi - for ((LOOPBACK_DEV=1;LOOPBACK_DEV<=${LOOPBACK_DEVS};LOOPBACK_DEV++)); do - sudo targetcli iscsi/iqn.2016-01.com.example:target/tpg1/luns/ create /backstores/fileio/loopback-${LOOPBACK_DEV} + + touch ${LOOPBACK_DEV_INFO} + JQ_OPT='.[0]' + COUNT=1 + let ITERATIONS=$(ls -1q $DEV_INFO_DIR | wc -l) + while [ $COUNT -lt "$ITERATIONS" ]; do + JQ_OPT="$JQ_OPT * .[$COUNT]" + COUNT=$[$COUNT+1] done - sudo targetcli iscsi/iqn.2016-01.com.example:target/tpg1/acls/ create $(sudo cat /etc/iscsi/initiatorname.iscsi | awk -F '=' '/^InitiatorName/ { print $NF}') - if [ "x$HOST_OS" == "xubuntu" ]; then - sudo targetcli iscsi/iqn.2016-01.com.example:target/tpg1 set attribute authentication=0 - fi - sudo iscsiadm --mode discovery --type sendtargets --portal 127.0.0.1 - sudo iscsiadm -m node -T iqn.2016-01.com.example:target -p 127.0.0.1:3260 -l - # Display disks - sudo lshw -class disk + + (cd $DEV_INFO_DIR; jq -s "$JQ_OPT" *) | json_to_yaml >> ${LOOPBACK_DEV_INFO} + cat ${LOOPBACK_DEV_INFO} } function ceph_support_install { diff --git a/tools/gate/funcs/kube.sh b/tools/gate/funcs/kube.sh index 7fe0fde878..0a9f11ebd2 100755 --- a/tools/gate/funcs/kube.sh +++ b/tools/gate/funcs/kube.sh @@ -143,7 +143,13 @@ function kubeadm_aio_clean { /var/lib/nfs-provisioner || true } -function ceph_kube_controller_manager_replace { - sudo docker pull ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE} - sudo docker tag ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE} ${BASE_KUBE_CONTROLLER_MANAGER_IMAGE} +function kube_label_node_block_devs { + for HOST in $(cat $LOOPBACK_DEV_INFO | yaml_to_json | jq -r ".block_devices | keys? | .[]"); do + for DEV_TYPE in $(cat $LOOPBACK_DEV_INFO | yaml_to_json | jq -r ".block_devices.\"$HOST\" | keys? | .[]"); do + DEV_ADDRS=$(cat $LOOPBACK_DEV_INFO | yaml_to_json | jq -r ".block_devices.\"$HOST\".\"$DEV_TYPE\" | .[]") + for DEV_ADDR in $(cat $LOOPBACK_DEV_INFO | yaml_to_json | jq -r ".block_devices.\"$HOST\".\"$DEV_TYPE\" | .[]"); do + kubectl label node $HOST device-$DEV_TYPE-$(echo $DEV_ADDR | tr '@' '_' | tr ':' '-' )=enabled + done + done + done } diff --git a/tools/gate/setup_gate.sh b/tools/gate/setup_gate.sh index 00fbabf4ed..399f67ca52 100755 --- a/tools/gate/setup_gate.sh +++ b/tools/gate/setup_gate.sh @@ -17,6 +17,7 @@ source ${WORK_DIR}/tools/gate/vars.sh source ${WORK_DIR}/tools/gate/funcs/common.sh source ${WORK_DIR}/tools/gate/funcs/network.sh source ${WORK_DIR}/tools/gate/funcs/helm.sh +source ${WORK_DIR}/tools/gate/funcs/kube.sh # Setup the logging location: by default use the working dir as the root. rm -rf ${LOGS_DIR} || true @@ -59,7 +60,10 @@ else bash ${WORK_DIR}/tools/gate/kubeadm_aio.sh bash ${WORK_DIR}/tools/gate/setup_gate_worker_nodes.sh fi - + if [ "x$LOOPBACK_CREATE" == "xtrue" ]; then + loopback_dev_info_collect + kube_label_node_block_devs + fi # Deploy OpenStack-Helm if ! [ "x$INTEGRATION_TYPE" == "x" ]; then bash ${WORK_DIR}/tools/gate/helm_dry_run.sh diff --git a/tools/gate/setup_gate_worker_nodes.sh b/tools/gate/setup_gate_worker_nodes.sh index 3788e3bba9..59c846550d 100755 --- a/tools/gate/setup_gate_worker_nodes.sh +++ b/tools/gate/setup_gate_worker_nodes.sh @@ -14,7 +14,6 @@ set -ex : ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"} source ${WORK_DIR}/tools/gate/vars.sh -export SUB_NODE_COUNT="$(($(echo ${SUB_NODE_IPS} | wc -w) + 1))" sudo chown $(whoami) ${SSH_PRIVATE_KEY} sudo chmod 600 ${SSH_PRIVATE_KEY} diff --git a/tools/gate/vars.sh b/tools/gate/vars.sh index a0d31bbc89..c80092f89c 100755 --- a/tools/gate/vars.sh +++ b/tools/gate/vars.sh @@ -62,14 +62,18 @@ export SERVICE_TEST_TIMEOUT=${SERVICE_TEST_TIMEOUT:="600"} # Setup Loopback device options export LOOPBACK_CREATE=${LOOPBACK_CREATE:="false"} -export LOOPBACK_DEVS=${LOOPBACK_DEVS:="3"} -export LOOPBACK_SIZE=${LOOPBACK_SIZE:="500M"} +export LOOPBACK_DEV_COUNT=${LOOPBACK_DEV_COUNT:="3,3,3"} +export LOOPBACK_SIZES=${LOOPBACK_SIZES:="8192M,1024M,1024M"} +export LOOPBACK_NAMES=${LOOPBACK_NAMES:="cephosd,cephjournal,swift"} export LOOPBACK_DIR=${LOOPBACK_DIR:="/var/lib/iscsi-loopback"} +export LOOPBACK_LOCAL_DISC_INFO=${LOOPBACK_LOCAL_DISC_INFO:="/tmp/loopback-local-disc-info"} +export LOOPBACK_DEV_INFO=${LOOPBACK_DEV_INFO:="/tmp/loopback-dev-info"} # Setup Multinode params export SSH_PRIVATE_KEY=${SSH_PRIVATE_KEY:="/etc/nodepool/id_rsa"} export PRIMARY_NODE_IP=${PRIMARY_NODE_IP:="$(cat /etc/nodepool/primary_node | tail -1)"} export SUB_NODE_IPS=${SUB_NODE_IPS:="$(cat /etc/nodepool/sub_nodes)"} +export SUB_NODE_COUNT="$(($(echo ${SUB_NODE_IPS} | wc -w) + 1))" # Define OpenStack Test Params export OSH_BR_EX_ADDR=${OSH_BR_EX_ADDR:="172.24.4.1/24"}