diff --git a/tools/deployment/multinode/030-ceph.sh b/tools/deployment/multinode/030-ceph.sh index f4419676e6..072ff5b844 100755 --- a/tools/deployment/multinode/030-ceph.sh +++ b/tools/deployment/multinode/030-ceph.sh @@ -19,20 +19,17 @@ set -xe #NOTE: Deploy command [ -s /tmp/ceph-fs-uuid.txt ] || uuidgen > /tmp/ceph-fs-uuid.txt CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" -CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" +CEPH_CLUSTER_NETWORK="${CEPH_PUBLIC_NETWORK}" CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" -#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this -# should be set to 'hammer' -. /etc/os-release -if [ "x${ID}" == "xubuntu" ] && \ - [ "$(uname -r | awk -F "." '{ print $2 }')" -lt "5" ]; then +#NOTE(portdirect): to use RBD devices with kernels < 4.5 this should be set to 'hammer' +LOWEST_CLUSTER_KERNEL_VERSION=$(kubectl get node -o go-template='{{range .items}}{{.status.nodeInfo.kernelVersion}}{{"\n"}}{{ end }}' | sort -V | tail -1) +if [ "$(echo ${LOWEST_CLUSTER_KERNEL_VERSION} | awk -F "." '{ print $1 }')" -lt "4" ] || [ "$(echo ${LOWEST_CLUSTER_KERNEL_VERSION} | awk -F "." '{ print $2 }')" -lt "15" ]; then + echo "Using hammer crush tunables" CRUSH_TUNABLES=hammer else CRUSH_TUNABLES=null fi -if [ "x${ID}" == "xcentos" ]; then - CRUSH_TUNABLES=hammer -fi +NUMBER_OF_OSDS="$(kubectl get nodes -l ceph-osd=enabled --no-headers | wc -l)" tee /tmp/ceph.yaml << EOF endpoints: ceph_mon: @@ -56,8 +53,7 @@ conf: crush: tunables: ${CRUSH_TUNABLES} target: - # NOTE(portdirect): 5 nodes, with one osd per node - osd: 5 + osd: ${NUMBER_OF_OSDS} pg_per_osd: 100 storage: osd: diff --git a/tools/deployment/multinode/040-ceph-ns-activate.sh b/tools/deployment/multinode/040-ceph-ns-activate.sh index f2a571449a..d19cca7809 100755 --- a/tools/deployment/multinode/040-ceph-ns-activate.sh +++ b/tools/deployment/multinode/040-ceph-ns-activate.sh @@ -18,7 +18,7 @@ set -xe #NOTE: Deploy command CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" -CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" +CEPH_CLUSTER_NETWORK="${CEPH_PUBLIC_NETWORK}" tee /tmp/ceph-openstack-config.yaml < $NODE_IPS -FIRST_IP_SUBNET=$(sudo docker run --rm ${UTILS_IMAGE} ipcalc "$(head -n 1 ${NODE_IPS})/24" | awk '/^Network/ { print $2 }') -LAST_IP_SUBNET=$(sudo docker run --rm ${UTILS_IMAGE} ipcalc "$(tail -n 1 ${NODE_IPS})/24" | awk '/^Network/ { print $2 }') +function run_and_log_ipcalc { + POD_NAME="tmp-$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-z | head -c 5; echo)" + kubectl run ${POD_NAME} \ + --generator=run-pod/v1 \ + --wait \ + --image ${UTILS_IMAGE} \ + --restart=Never \ + ipcalc -- "$1" + end=$(($(date +%s) + 900)) + until kubectl get pod/${POD_NAME} -o go-template='{{.status.phase}}' | grep -q Succeeded; do + now=$(date +%s) + [ $now -gt $end ] && echo containers failed to start. && \ + kubectl get pod/${POD_NAME} -o wide && exit 1 + done + kubectl logs pod/${POD_NAME} + kubectl delete pod/${POD_NAME} +} +FIRST_IP_SUBNET=$(run_and_log_ipcalc "$(head -n 1 ${NODE_IPS})/24" | awk '/^Network/ { print $2 }') +LAST_IP_SUBNET=$(run_and_log_ipcalc "$(tail -n 1 ${NODE_IPS})/24" | awk '/^Network/ { print $2 }') rm -f $NODE_IPS function ip_diff { echo $(($(echo $LAST_IP_SUBNET | awk -F '.' "{ print \$$1}") - $(echo $FIRST_IP_SUBNET | awk -F '.' "{ print \$$1}")))