diff --git a/doc/source/install/developer/cleaning-deployment.rst b/doc/source/install/developer/cleaning-deployment.rst index c802bd2bc0..dad56b2db3 100644 --- a/doc/source/install/developer/cleaning-deployment.rst +++ b/doc/source/install/developer/cleaning-deployment.rst @@ -54,6 +54,11 @@ containers before removing the directories used on the host by pods. sudo rm -rf /var/lib/nova/* sudo rm -rf /var/lib/libvirt/* sudo rm -rf /etc/libvirt/qemu/* + #NOTE(chinasubbareddy) cleanup LVM volume groups in case of disk backed ceph osd deployments + for VG in `vgs|grep -v VG|grep -i ceph|awk '{print $1}'`; do + echo $VG + vgremove -y $VG + done # NOTE(portdirect): Clean up mounts left behind by kubernetes pods sudo findmnt --raw | awk '/^\/var\/lib\/kubelet\/pods/ { print $1 }' | xargs -r -L1 -P16 sudo umount -f -l diff --git a/doc/source/install/multinode.rst b/doc/source/install/multinode.rst index 49c266c7ae..6b43bf1636 100644 --- a/doc/source/install/multinode.rst +++ b/doc/source/install/multinode.rst @@ -104,11 +104,20 @@ Alternatively, this step can be performed by running the script directly: OSH_DEPLOY_MULTINODE=True ./tools/deployment/component/common/ingress.sh +Create loopback devices for CEPH +-------------------------------- + +Create two loopback devices for ceph as one disk for OSD data and other disk for +block DB and block WAL. + +.. code-block:: shell + + ansible all -i /opt/openstack-helm-infra/tools/gate/devel/multinode-inventory.yaml -m shell -s -a "/opt/openstack-helm/tools/deployment/common/setup-ceph-loopback-device.sh" Deploy Ceph ----------- -The script below configures Ceph to use filesystem directory-based storage. +The script below configures Ceph to use loopback devices created in previous step as backend for ceph osds. To configure a custom block device-based backend, please refer to the ``ceph-osd`` `values.yaml `_. diff --git a/tools/deployment/common/setup-ceph-loopback-device.sh b/tools/deployment/common/setup-ceph-loopback-device.sh new file mode 100755 index 0000000000..ca81c604e7 --- /dev/null +++ b/tools/deployment/common/setup-ceph-loopback-device.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -xe + +sudo df -lh +sudo lsblk +sudo mkdir -p /var/lib/openstack-helm/ceph +sudo truncate -s 10G /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img +sudo truncate -s 8G /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img +sudo losetup /dev/loop0 /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img +sudo losetup /dev/loop1 /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img +#lets verify the devices +sudo df -lh +sudo lsblk diff --git a/tools/deployment/component/ceph/ceph.sh b/tools/deployment/component/ceph/ceph.sh index 99dee4efff..1763740edc 100755 --- a/tools/deployment/component/ceph/ceph.sh +++ b/tools/deployment/component/ceph/ceph.sh @@ -154,17 +154,21 @@ conf: storage: osd: - data: - type: directory - location: /var/lib/openstack-helm/ceph/osd/osd-one - journal: - type: directory - location: /var/lib/openstack-helm/ceph/osd/journal-one - + type: bluestore + location: /dev/loop0 + block_db: + location: /dev/loop1 + size: "5GB" + block_wal: + location: /dev/loop1 + size: "2GB" pod: replicas: mds: 1 mgr: 1 rgw: 1 +deploy: + tool: "ceph-volume" EOF diff --git a/tools/deployment/developer/ceph/040-ceph.sh b/tools/deployment/developer/ceph/040-ceph.sh index 1e15d60cd8..39cacf7904 100755 --- a/tools/deployment/developer/ceph/040-ceph.sh +++ b/tools/deployment/developer/ceph/040-ceph.sh @@ -14,6 +14,18 @@ set -xe +# Create loop back devices for ceph osds. +sudo df -lh +sudo lsblk +sudo mkdir -p /var/lib/openstack-helm/ceph +sudo truncate -s 10G /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img +sudo truncate -s 8G /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img +sudo losetup /dev/loop0 /var/lib/openstack-helm/ceph/ceph-osd-data-loopbackfile.img +sudo losetup /dev/loop1 /var/lib/openstack-helm/ceph/ceph-osd-db-wal-loopbackfile.img +# lets check the devices +sudo df -lh +sudo lsblk + #NOTE: Lint and package chart export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${OSH_INFRA_PATH:="../openstack-helm-infra"}"}" for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do @@ -143,13 +155,16 @@ conf: replication: 1 percent_total_data: 34.8 storage: - osd: + osd: - data: - type: directory - location: /var/lib/openstack-helm/ceph/osd/osd-one - journal: - type: directory - location: /var/lib/openstack-helm/ceph/osd/journal-one + type: bluestore + location: /dev/loop0 + block_db: + location: /dev/loop1 + size: "5GB" + block_wal: + location: /dev/loop1 + size: "2GB" pod: replicas: diff --git a/tools/deployment/multinode/030-ceph.sh b/tools/deployment/multinode/030-ceph.sh index 9aa62f93dc..57b3a9c82f 100755 --- a/tools/deployment/multinode/030-ceph.sh +++ b/tools/deployment/multinode/030-ceph.sh @@ -56,11 +56,14 @@ conf: storage: osd: - data: - type: directory - location: /var/lib/openstack-helm/ceph/osd/osd-one - journal: - type: directory - location: /var/lib/openstack-helm/ceph/osd/journal-one + type: bluestore + location: /dev/loop0 + block_db: + location: /dev/loop1 + size: "5GB" + block_wal: + location: /dev/loop1 + size: "2GB" storageclass: cephfs: provision_storage_class: false diff --git a/zuul.d/jobs-openstack-helm.yaml b/zuul.d/jobs-openstack-helm.yaml index fc7f9ef058..859d20760c 100644 --- a/zuul.d/jobs-openstack-helm.yaml +++ b/zuul.d/jobs-openstack-helm.yaml @@ -85,6 +85,7 @@ container_distro_version: bionic gate_scripts: - ./tools/deployment/common/install-packages.sh + - ./tools/deployment/common/setup-ceph-loopback-device.sh - ./tools/deployment/common/deploy-k8s.sh - ./tools/deployment/common/setup-client.sh - ./tools/scripts/tls/cert-manager.sh @@ -309,6 +310,7 @@ feature_gates: apparmor gate_scripts: - ./tools/deployment/common/install-packages.sh + - ./tools/deployment/common/setup-ceph-loopback-device.sh - ./tools/deployment/common/deploy-k8s.sh - ./tools/deployment/common/setup-client.sh - ./tools/deployment/component/ceph/ceph.sh @@ -399,6 +401,7 @@ feature_gates: netpol gate_scripts: - ./tools/deployment/common/install-packages.sh + - ./tools/deployment/common/setup-ceph-loopback-device.sh - ./tools/deployment/common/deploy-k8s.sh - ./tools/deployment/common/setup-client.sh - ./tools/deployment/component/ceph/ceph.sh