diff --git a/tools/deployment/apparmor/019-setup-ceph-loopback-device.sh b/tools/deployment/apparmor/019-setup-ceph-loopback-device.sh deleted file mode 120000 index 3d8509fcc..000000000 --- a/tools/deployment/apparmor/019-setup-ceph-loopback-device.sh +++ /dev/null @@ -1 +0,0 @@ -../common/019-setup-ceph-loopback-device.sh \ No newline at end of file diff --git a/tools/deployment/common/019-setup-ceph-loopback-device.sh b/tools/deployment/common/019-setup-ceph-loopback-device.sh deleted file mode 100755 index d40cf5329..000000000 --- a/tools/deployment/common/019-setup-ceph-loopback-device.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -xe -sudo df -lh -sudo lsblk -sudo mkdir -p /var/lib/openstack-helm/ceph -sudo truncate -s 10G /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img -sudo truncate -s 8G /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img -sudo losetup /dev/loop0 /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img -sudo losetup /dev/loop1 /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img -#lets check the devices -sudo df -lh -sudo lsblk diff --git a/tools/deployment/common/setup-ceph-loopback-device.sh b/tools/deployment/common/setup-ceph-loopback-device.sh new file mode 100755 index 000000000..67dc6d795 --- /dev/null +++ b/tools/deployment/common/setup-ceph-loopback-device.sh @@ -0,0 +1,47 @@ +#!/bin/bash +function setup_loopback_devices() { + osd_data_device="$1" + osd_wal_db_device="$2" + namespace=${CEPH_NAMESPACE} + sudo mkdir -p /var/lib/openstack-helm/$namespace + sudo truncate -s 10G /var/lib/openstack-helm/$namespace/ceph-osd-data-loopbackfile.img + sudo truncate -s 8G /var/lib/openstack-helm/$namespace/ceph-osd-db-wal-loopbackfile.img + sudo losetup $osd_data_device /var/lib/openstack-helm/$namespace/ceph-osd-data-loopbackfile.img + sudo losetup $osd_wal_db_device /var/lib/openstack-helm/$namespace/ceph-osd-db-wal-loopbackfile.img + #lets verify the devices + sudo losetup -a +} + +while [[ "$#" > 0 ]]; do case $1 in + -d|--ceph-osd-data) OSD_DATA_DEVICE="$2"; shift;shift;; + -w|--ceph-osd-dbwal) OSD_DB_WAL_DEVICE="$2";shift;shift;; + -v|--verbose) VERBOSE=1;shift;; + *) echo "Unknown parameter passed: $1"; shift;; +esac; done + +# verify params +if [ -z "$OSD_DATA_DEVICE" ]; then + OSD_DATA_DEVICE=/dev/loop0 + echo "Ceph osd data device is not set so using ${OSD_DATA_DEVICE}" +else + ceph_osd_disk_name=`basename "$OSD_DATA_DEVICE"` + if losetup -a|grep $ceph_osd_disk_name; then + echo "Ceph osd data device is already in use, please double check and correct the device name" + exit 1 + fi +fi + +if [ -z "$OSD_DB_WAL_DEVICE" ]; then + OSD_DB_WAL_DEVICE=/dev/loop1 + echo "Ceph osd db/wal device is not set so using ${OSD_DB_WAL_DEVICE}" +else + ceph_dbwal_disk_name=`basename "$OSD_DB_WAL_DEVICE"` + if losetup -a|grep $ceph_dbwal_disk_name; then + echo "Ceph osd dbwal device is already in use, please double check and correct the device name" + exit 1 + fi +fi + +: "${CEPH_NAMESPACE:="ceph"}" +# setup loopback devices for ceph osds +setup_loopback_devices $OSD_DATA_DEVICE $OSD_DB_WAL_DEVICE diff --git a/tools/deployment/elastic-beats/019-setup-ceph-loopback-device.sh b/tools/deployment/elastic-beats/019-setup-ceph-loopback-device.sh deleted file mode 120000 index 436c5d6e2..000000000 --- a/tools/deployment/elastic-beats/019-setup-ceph-loopback-device.sh +++ /dev/null @@ -1 +0,0 @@ -../multinode/019-setup-ceph-loopback-device.sh \ No newline at end of file diff --git a/tools/deployment/multinode/019-setup-ceph-loopback-device.sh b/tools/deployment/multinode/019-setup-ceph-loopback-device.sh deleted file mode 100755 index 250ac8363..000000000 --- a/tools/deployment/multinode/019-setup-ceph-loopback-device.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -xe -sudo df -lh -sudo lsblk -sudo mkdir -p /var/lib/openstack-helm/ceph -sudo truncate -s 10G /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img -sudo truncate -s 8G /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img -sudo losetup /dev/loop0 /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img -sudo losetup /dev/loop1 /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img -# lets check the devices -sudo df -lh -sudo lsblk diff --git a/tools/deployment/multinode/030-ceph.sh b/tools/deployment/multinode/030-ceph.sh index 5f61d0963..1743f5be2 100755 --- a/tools/deployment/multinode/030-ceph.sh +++ b/tools/deployment/multinode/030-ceph.sh @@ -14,6 +14,10 @@ set -xe +# setup loopback devices for ceph +./tools/deployment/common/setup-ceph-loopback-device.sh --ceph-osd-data \ +${CEPH_OSD_DATA_DEVICE:=/dev/loop0} --ceph-osd-dbwal ${CEPH_OSD_DB_WAL_DEVICE:=/dev/loop1} + #NOTE: Lint and package chart make ceph-mon make ceph-osd @@ -25,14 +29,17 @@ make ceph-provisioners CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" CEPH_CLUSTER_NETWORK="${CEPH_PUBLIC_NETWORK}" CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" + #NOTE(portdirect): to use RBD devices with kernels < 4.5 this should be set to 'hammer' -LOWEST_CLUSTER_KERNEL_VERSION=$(kubectl get node -o go-template='{{range .items}}{{.status.nodeInfo.kernelVersion}}{{"\n"}}{{ end }}' | sort -V | tail -1) -if [ "$(echo ${LOWEST_CLUSTER_KERNEL_VERSION} | awk -F "." '{ print $1 }')" -lt "4" ] || [ "$(echo ${LOWEST_CLUSTER_KERNEL_VERSION} | awk -F "." '{ print $2 }')" -lt "15" ]; then - echo "Using hammer crush tunables" +. /etc/os-release +if [ "x${ID}" == "xcentos" ] || \ + ([ "x${ID}" == "xubuntu" ] && \ + dpkg --compare-versions "$(uname -r)" "lt" "4.5"); then CRUSH_TUNABLES=hammer else CRUSH_TUNABLES=null fi + NUMBER_OF_OSDS="$(kubectl get nodes -l ceph-osd=enabled --no-headers | wc -l)" tee /tmp/ceph.yaml << EOF endpoints: @@ -70,12 +77,12 @@ conf: osd: - data: type: bluestore - location: /dev/loop0 + location: ${CEPH_OSD_DATA_DEVICE} block_db: - location: /dev/loop1 + location: ${CEPH_OSD_DB_WAL_DEVICE} size: "5GB" block_wal: - location: /dev/loop1 + location: ${CEPH_OSD_DB_WAL_DEVICE} size: "2GB" jobs: diff --git a/tools/deployment/openstack-support/019-setup-ceph-loopback-device.sh b/tools/deployment/openstack-support/019-setup-ceph-loopback-device.sh deleted file mode 120000 index 3d8509fcc..000000000 --- a/tools/deployment/openstack-support/019-setup-ceph-loopback-device.sh +++ /dev/null @@ -1 +0,0 @@ -../common/019-setup-ceph-loopback-device.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/019-setup-ceph-loopback-device.sh b/tools/deployment/osh-infra-kafka/019-setup-ceph-loopback-device.sh deleted file mode 120000 index 3d8509fcc..000000000 --- a/tools/deployment/osh-infra-kafka/019-setup-ceph-loopback-device.sh +++ /dev/null @@ -1 +0,0 @@ -../common/019-setup-ceph-loopback-device.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-logging/019-setup-ceph-loopback-device.sh b/tools/deployment/osh-infra-logging/019-setup-ceph-loopback-device.sh deleted file mode 120000 index 3d8509fcc..000000000 --- a/tools/deployment/osh-infra-logging/019-setup-ceph-loopback-device.sh +++ /dev/null @@ -1 +0,0 @@ -../common/019-setup-ceph-loopback-device.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-logging/020-ceph.sh b/tools/deployment/osh-infra-logging/020-ceph.sh index a2f3a6a23..c60fd6636 100755 --- a/tools/deployment/osh-infra-logging/020-ceph.sh +++ b/tools/deployment/osh-infra-logging/020-ceph.sh @@ -14,6 +14,10 @@ set -xe +# setup loopback devices for ceph +./tools/deployment/common/setup-ceph-loopback-device.sh --ceph-osd-data \ +${CEPH_OSD_DATA_DEVICE:=/dev/loop0} --ceph-osd-dbwal ${CEPH_OSD_DB_WAL_DEVICE:=/dev/loop1} + #NOTE: Lint and package chart for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do make "${CHART}" @@ -26,8 +30,9 @@ CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" #NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this # should be set to 'hammer' . /etc/os-release -if [ "x${ID}" == "xubuntu" ] && \ - [ "$(uname -r | awk -F "." '{ print $2 }')" -lt "5" ]; then +if [ "x${ID}" == "xcentos" ] || \ + ([ "x${ID}" == "xubuntu" ] && \ + dpkg --compare-versions "$(uname -r)" "lt" "4.5"); then CRUSH_TUNABLES=hammer else CRUSH_TUNABLES=null @@ -160,12 +165,12 @@ conf: osd: - data: type: bluestore - location: /dev/loop0 + location: ${CEPH_OSD_DATA_DEVICE} block_db: - location: /dev/loop1 + location: ${CEPH_OSD_DB_WAL_DEVICE} size: "5GB" block_wal: - location: /dev/loop1 + location: ${CEPH_OSD_DB_WAL_DEVICE} size: "2GB" pod: diff --git a/tools/deployment/tenant-ceph/019-setup-ceph-loopback-device.sh b/tools/deployment/tenant-ceph/019-setup-ceph-loopback-device.sh deleted file mode 100755 index 5eba99790..000000000 --- a/tools/deployment/tenant-ceph/019-setup-ceph-loopback-device.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -xe -sudo df -lh -sudo lsblk -sudo mkdir -p /var/lib/openstack-helm/ceph -sudo truncate -s 10G /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img -sudo truncate -s 8G /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img -sudo losetup /dev/loop0 /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img -sudo losetup /dev/loop1 /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img - -#second disk for tenant-ceph -sudo mkdir -p /var/lib/openstack-helm/tenant-ceph -sudo truncate -s 10G /var/lib/openstack-helm/tenant-ceph/ceph-osd-data-loopbackfile.img -sudo truncate -s 8G /var/lib/openstack-helm/tenant-ceph/ceph-osd-db-wal-loopbackfile.img -sudo losetup /dev/loop2 /var/lib/openstack-helm/tenant-ceph/ceph-osd-data-loopbackfile.img -sudo losetup /dev/loop3 /var/lib/openstack-helm/tenant-ceph/ceph-osd-db-wal-loopbackfile.img - -# lets check the devices -sudo df -lh -sudo lsblk diff --git a/tools/deployment/tenant-ceph/030-ceph.sh b/tools/deployment/tenant-ceph/030-ceph.sh index b74b09d21..b4470a91a 100755 --- a/tools/deployment/tenant-ceph/030-ceph.sh +++ b/tools/deployment/tenant-ceph/030-ceph.sh @@ -14,6 +14,10 @@ set -xe +# setup loopback devices for ceph +./tools/deployment/common/setup-ceph-loopback-device.sh --ceph-osd-data \ +${CEPH_OSD_DATA_DEVICE:=/dev/loop0} --ceph-osd-dbwal ${CEPH_OSD_DB_WAL_DEVICE:=/dev/loop1} + #NOTE: Deploy command [ -s /tmp/ceph-fs-uuid.txt ] || uuidgen > /tmp/ceph-fs-uuid.txt CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" @@ -22,8 +26,9 @@ CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" #NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this # should be set to 'hammer' . /etc/os-release -if [ "x${ID}" == "xubuntu" ] && \ - [ "$(uname -r | awk -F "." '{ print $2 }')" -lt "5" ]; then +if [ "x${ID}" == "xcentos" ] || \ + ([ "x${ID}" == "xubuntu" ] && \ + dpkg --compare-versions "$(uname -r)" "lt" "4.5"); then CRUSH_TUNABLES=hammer else CRUSH_TUNABLES=null @@ -94,12 +99,12 @@ conf: osd: - data: type: bluestore - location: /dev/loop0 + location: ${CEPH_OSD_DATA_DEVICE} block_db: - location: /dev/loop1 + location: ${CEPH_OSD_DB_WAL_DEVICE} size: "5GB" block_wal: - location: /dev/loop1 + location: ${CEPH_OSD_DB_WAL_DEVICE} size: "2GB" storageclass: rbd: diff --git a/tools/deployment/tenant-ceph/040-tenant-ceph.sh b/tools/deployment/tenant-ceph/040-tenant-ceph.sh index 76539fce3..842a047a6 100755 --- a/tools/deployment/tenant-ceph/040-tenant-ceph.sh +++ b/tools/deployment/tenant-ceph/040-tenant-ceph.sh @@ -14,6 +14,15 @@ set -xe +: "${CEPH_OSD_DATA_DEVICE:=/dev/loop2}" +: "${CEPH_OSD_DB_WAL_DEVICE:=/dev/loop3}" +# setup loopback devices for ceph +export CEPH_NAMESPACE="tenant-ceph" +./tools/deployment/common/setup-ceph-loopback-device.sh --ceph-osd-data ${CEPH_OSD_DATA_DEVICE} --ceph-osd-dbwal ${CEPH_OSD_DB_WAL_DEVICE} + +# setup loopback devices for ceph osds +setup_loopback_devices $OSD_DATA_DEVICE $OSD_DB_WAL_DEVICE + #NOTE: Deploy command [ -s /tmp/tenant-ceph-fs-uuid.txt ] || uuidgen > /tmp/tenant-ceph-fs-uuid.txt CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" @@ -132,12 +141,12 @@ conf: osd: - data: type: bluestore - location: /dev/loop2 + location: ${CEPH_OSD_DATA_DEVICE} block_db: - location: /dev/loop3 + location: ${CEPH_OSD_DB_WAL_DEVICE} size: "5GB" block_wal: - location: /dev/loop3 + location: ${CEPH_OSD_DB_WAL_DEVICE} size: "2GB" mon: directory: /var/lib/openstack-helm/tenant-ceph/mon diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 46f88f99d..04f96028a 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -60,7 +60,6 @@ vars: gate_scripts_relative_path: ../openstack-helm-infra gate_scripts: - - ./tools/deployment/multinode/019-setup-ceph-loopback-device.sh - ./tools/deployment/multinode/010-deploy-docker-registry.sh - ./tools/deployment/multinode/020-ingress.sh - ./tools/deployment/multinode/030-ceph.sh @@ -103,7 +102,6 @@ vars: gate_scripts_relative_path: ../openstack-helm-infra gate_scripts: - - ./tools/deployment/tenant-ceph/019-setup-ceph-loopback-device.sh - ./tools/deployment/tenant-ceph/010-relabel-nodes.sh - ./tools/deployment/tenant-ceph/020-ingress.sh - ./tools/deployment/tenant-ceph/030-ceph.sh @@ -141,7 +139,6 @@ gate_scripts_relative_path: ../openstack-helm-infra gate_scripts: - ./tools/deployment/osh-infra-logging/000-install-packages.sh - - ./tools/deployment/osh-infra-logging/019-setup-ceph-loopback-device.sh - ./tools/deployment/osh-infra-logging/005-deploy-k8s.sh - ./tools/deployment/osh-infra-logging/010-ingress.sh - ./tools/deployment/osh-infra-logging/020-ceph.sh @@ -166,7 +163,6 @@ gate_scripts_relative_path: ../openstack-helm-infra gate_scripts: - ./tools/deployment/osh-infra-kafka/000-install-packages.sh - - ./tools/deployment/osh-infra-kafka/019-setup-ceph-loopback-device.sh - ./tools/deployment/osh-infra-kafka/005-deploy-k8s.sh - ./tools/deployment/osh-infra-kafka/010-ingress.sh - ./tools/deployment/osh-infra-kafka/020-ceph.sh @@ -281,7 +277,6 @@ gate_scripts_relative_path: ../openstack-helm-infra gate_scripts: - ./tools/deployment/apparmor/000-install-packages.sh - - ./tools/deployment/apparmor/019-setup-ceph-loopback-device.sh - ./tools/deployment/apparmor/001-setup-apparmor-profiles.sh - ./tools/deployment/apparmor/005-deploy-k8s.sh - ./tools/deployment/apparmor/015-ingress.sh @@ -316,7 +311,6 @@ gate_scripts_relative_path: ../openstack-helm-infra gate_scripts: - ./tools/deployment/osh-infra-logging/000-install-packages.sh - - ./tools/deployment/osh-infra-logging/019-setup-ceph-loopback-device.sh - ./tools/deployment/osh-infra-logging/005-deploy-k8s.sh - ./tools/deployment/osh-infra-logging/010-ingress.sh - ./tools/deployment/osh-infra-logging/020-ceph.sh @@ -348,7 +342,6 @@ gate_scripts_relative_path: ../openstack-helm-infra gate_scripts: - ./tools/deployment/openstack-support/000-install-packages.sh - - ./tools/deployment/openstack-support/019-setup-ceph-loopback-device.sh - ./tools/deployment/openstack-support/005-deploy-k8s.sh - ./tools/deployment/openstack-support/007-namespace-config.sh - ./tools/deployment/openstack-support/010-ingress.sh @@ -401,7 +394,6 @@ gate_scripts_relative_path: ../openstack-helm-infra gate_scripts: - ./tools/deployment/openstack-support/000-install-packages.sh - - ./tools/deployment/openstack-support/019-setup-ceph-loopback-device.sh - ./tools/deployment/openstack-support/005-deploy-k8s.sh - ./tools/deployment/openstack-support/007-namespace-config.sh - ./tools/deployment/openstack-support/010-ingress.sh @@ -469,7 +461,6 @@ vars: gate_scripts_relative_path: ../openstack-helm-infra gate_scripts: - - ./tools/deployment/elastic-beats/019-setup-ceph-loopback-device.sh - ./tools/deployment/elastic-beats/005-deploy-k8s.sh - ./tools/deployment/elastic-beats/020-ingress.sh - ./tools/deployment/elastic-beats/030-ceph.sh