From d95259936f7a2af83b279c32dff2e3d62bbd08ab Mon Sep 17 00:00:00 2001 From: Tin Lam Date: Mon, 25 May 2020 21:08:54 +0000 Subject: [PATCH] Revert "[ceph-osd, ceph-client] Weight OSDs as they are added" This patch currently breaks cinder helm test in the OSH cinder jobs blocking the gate. Proposing to revert to unblock the jobs. This reverts commit f59cb11932e30bb607a580c976871cdecd7a714c. Change-Id: I73012ec6f4c3d751131f1c26eea9266f7abc1809 --- ceph-client/templates/bin/pool/_init.sh.tpl | 27 +++++++------------ ceph-client/templates/job-rbd-pool.yaml | 4 --- ceph-client/values.yaml | 4 --- .../bin/osd/ceph-volume/_block.sh.tpl | 4 +-- .../bin/osd/ceph-volume/_bluestore.sh.tpl | 4 +-- .../bin/osd/ceph-volume/_common.sh.tpl | 21 --------------- 6 files changed, 13 insertions(+), 51 deletions(-) diff --git a/ceph-client/templates/bin/pool/_init.sh.tpl b/ceph-client/templates/bin/pool/_init.sh.tpl index 6ce3d23cf..bf8c44c65 100644 --- a/ceph-client/templates/bin/pool/_init.sh.tpl +++ b/ceph-client/templates/bin/pool/_init.sh.tpl @@ -67,6 +67,13 @@ create_crushrule {{ .name }} {{ .crush_rule }} {{ .failure_domain }} {{ .device_ {{- end }} {{- end }} +function reweight_osds () { + for OSD_ID in $(ceph --cluster "${CLUSTER}" osd df | awk '$3 == "0" {print $1}'); do + OSD_WEIGHT=$(ceph --cluster "${CLUSTER}" osd df --format json-pretty| grep -A7 "\bosd.${OSD_ID}\b" | awk '/"kb"/{ gsub(",",""); d= $2/1073741824 ; r = sprintf("%.2f", d); print r }'); + ceph --cluster "${CLUSTER}" osd crush reweight osd.${OSD_ID} ${OSD_WEIGHT}; + done +} + function enable_autoscaling () { if [[ "${ENABLE_AUTOSCALER}" == "true" ]]; then ceph mgr module enable pg_autoscaler @@ -74,22 +81,6 @@ function enable_autoscaling () { fi } -function set_cluster_flags () { - if [[ ! -z "${CLUSTER_SET_FLAGS}" ]]; then - for flag in ${CLUSTER_SET_FLAGS}; do - ceph osd set ${flag} - done - fi -} - -function unset_cluster_flags () { - if [[ ! -z "${CLUSTER_UNSET_FLAGS}" ]]; then - for flag in ${CLUSTER_UNSET_FLAGS}; do - ceph osd unset ${flag} - done - fi -} - function create_pool () { POOL_APPLICATION=$1 POOL_NAME=$2 @@ -171,6 +162,8 @@ function manage_pool () { ceph --cluster "${CLUSTER}" osd pool set-quota "${POOL_NAME}" max_bytes $POOL_QUOTA } +reweight_osds + {{ $targetPGperOSD := .Values.conf.pool.target.pg_per_osd }} {{ $crushRuleDefault := .Values.conf.pool.default.crush_rule }} {{ $targetQuota := .Values.conf.pool.target.quota | default 100 }} @@ -182,8 +175,6 @@ if [[ -z "$(ceph osd versions | grep ceph\ version | grep -v nautilus)" ]]; then else cluster_capacity=$(ceph --cluster "${CLUSTER}" df | head -n3 | tail -n1 | awk '{print $1 substr($2, 1, 1)}' | numfmt --from=iec) fi -set_cluster_flags -unset_cluster_flags {{- range $pool := .Values.conf.pool.spec -}} {{- with $pool }} {{- if .crush_rule }} diff --git a/ceph-client/templates/job-rbd-pool.yaml b/ceph-client/templates/job-rbd-pool.yaml index 351ef761d..47c8bc947 100644 --- a/ceph-client/templates/job-rbd-pool.yaml +++ b/ceph-client/templates/job-rbd-pool.yaml @@ -52,10 +52,6 @@ spec: value: "ceph" - name: ENABLE_AUTOSCALER value: {{ .Values.conf.features.pg_autoscaler | quote }} - - name: CLUSTER_SET_FLAGS - value: {{ .Values.conf.features.cluster_flags.set | quote }} - - name: CLUSTER_UNSET_FLAGS - value: {{ .Values.conf.features.cluster_flags.unset | quote }} command: - /tmp/pool-init.sh volumeMounts: diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml index 06ee6e77b..534fb1314 100644 --- a/ceph-client/values.yaml +++ b/ceph-client/values.yaml @@ -254,10 +254,6 @@ conf: mds: true mgr: true pg_autoscaler: true - cluster_flags: - # List of flags to set or unset separated by spaces - set: "" - unset: "" pool: #NOTE(portdirect): this drives a simple approximation of # https://ceph.com/pgcalc/, the `target.osd` key should be set to match the diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_block.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_block.sh.tpl index 7ccb8e1fe..68e150efb 100644 --- a/ceph-osd/templates/bin/osd/ceph-volume/_block.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-volume/_block.sh.tpl @@ -83,8 +83,8 @@ else --no-systemd ${OSD_ID} ${OSD_FSID} fi -# NOTE(stevetaylor): Set the OSD's crush weight (use noin flag to prevent rebalancing if necessary) -OSD_WEIGHT=$(get_osd_crush_weight_from_device ${OSD_DEVICE}) +# NOTE(supamatt): set the initial crush weight of the OSD to 0 to prevent automatic rebalancing +OSD_WEIGHT=0 # NOTE(supamatt): add or move the OSD's CRUSH location crush_location diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl index a3110ac56..80a16bbeb 100644 --- a/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl @@ -89,8 +89,8 @@ else fi fi -# NOTE(stevetaylor): Set the OSD's crush weight (use noin flag to prevent rebalancing if necessary) -OSD_WEIGHT=$(get_osd_crush_weight_from_device ${OSD_DEVICE}) +# NOTE(supamatt): set the initial crush weight of the OSD to 0 to prevent automatic rebalancing +OSD_WEIGHT=0 # NOTE(supamatt): add or move the OSD's CRUSH location crush_location diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl index e8c692640..a1f61c50e 100644 --- a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl +++ b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl @@ -276,27 +276,6 @@ function get_lvm_tag_from_device { get_lvm_tag_from_volume ${logical_volume} ${tag} } -# Helper function to get the size of a logical volume -function get_lv_size_from_device { - device="$1" - logical_volume="$(get_lv_from_device ${device})" - - lvs ${logical_volume} -o LV_SIZE --noheadings --units k --nosuffix | xargs | cut -d'.' -f1 -} - -# Helper function to get the crush weight for an osd device -function get_osd_crush_weight_from_device { - device="$1" - lv_size="$(get_lv_size_from_device ${device})" # KiB - - if [[ ! -z "${BLOCK_DB_SIZE}" ]]; then - db_size=$(echo "${BLOCK_DB_SIZE}" | cut -d'B' -f1 | numfmt --from=iec | awk '{print $1/1024}') # KiB - lv_size=$((lv_size+db_size)) # KiB - fi - - echo ${lv_size} | awk '{printf("%.2f\n", $1/1073741824)}' # KiB to TiB -} - # Helper function to get a cluster FSID from a physical device function get_cluster_fsid_from_device { device="$1"