diff --git a/ceph-client/Chart.yaml b/ceph-client/Chart.yaml index ab237d0a3..6c38f1e55 100644 --- a/ceph-client/Chart.yaml +++ b/ceph-client/Chart.yaml @@ -15,6 +15,6 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Ceph Client name: ceph-client -version: 0.1.10 +version: 0.1.11 home: https://github.com/ceph/ceph-client ... diff --git a/ceph-client/templates/bin/pool/_init.sh.tpl b/ceph-client/templates/bin/pool/_init.sh.tpl index bfa3fa2f5..2e338d95e 100644 --- a/ceph-client/templates/bin/pool/_init.sh.tpl +++ b/ceph-client/templates/bin/pool/_init.sh.tpl @@ -148,10 +148,15 @@ function reweight_osds () { function enable_or_disable_autoscaling () { if [[ "${ENABLE_AUTOSCALER}" == "true" ]]; then - ceph mgr module enable pg_autoscaler + if [[ $(ceph mgr versions | awk '/version/{print $3}' | cut -d. -f1) -eq 14 ]]; then + ceph mgr module enable pg_autoscaler # only required for nautilus + fi ceph config set global osd_pool_default_pg_autoscale_mode on else - ceph mgr module disable pg_autoscaler + if [[ $(ceph mgr versions | awk '/version/{print $3}' | cut -d. -f1) -eq 14 ]]; then + ceph mgr module disable pg_autoscaler # only required for nautilus + fi + ceph config set global osd_pool_default_pg_autoscale_mode off fi } @@ -178,16 +183,30 @@ function create_pool () { POOL_PLACEMENT_GROUPS=$4 POOL_CRUSH_RULE=$5 POOL_PROTECTION=$6 + PG_NUM_MIN={{.Values.conf.pool.target.pg_num_min}} if ! ceph --cluster "${CLUSTER}" osd pool stats "${POOL_NAME}" > /dev/null 2>&1; then - ceph --cluster "${CLUSTER}" osd pool create "${POOL_NAME}" ${POOL_PLACEMENT_GROUPS} + if [[ ${POOL_PLACEMENT_GROUPS} -gt 0 ]]; then + ceph --cluster "${CLUSTER}" osd pool create "${POOL_NAME}" ${POOL_PLACEMENT_GROUPS} + else + ceph --cluster "${CLUSTER}" osd pool create "${POOL_NAME}" ${PG_NUM_MIN} --pg-num-min ${PG_NUM_MIN} + fi while [ $(ceph --cluster "${CLUSTER}" -s | grep creating -c) -gt 0 ]; do echo -n .;sleep 1; done ceph --cluster "${CLUSTER}" osd pool application enable "${POOL_NAME}" "${POOL_APPLICATION}" fi - if [[ $(ceph osd versions | awk '/version/{print $3}' | cut -d. -f1) -ge 14 ]] && [[ "${ENABLE_AUTOSCALER}" == "true" ]] ; then - ceph --cluster "${CLUSTER}" osd pool set "${POOL_NAME}" pg_autoscale_mode on - else - ceph --cluster "${CLUSTER}" osd pool set "${POOL_NAME}" pg_autoscale_mode off + if [[ $(ceph mgr versions | awk '/version/{print $3}' | cut -d. -f1) -ge 14 ]]; then + if [[ "${ENABLE_AUTOSCALER}" == "true" ]]; then + pool_values=$(ceph --cluster "${CLUSTER}" osd pool get "${POOL_NAME}" all -f json) + pg_num=$(jq '.pg_num' <<< "${pool_values}") + pg_num_min=$(jq '.pg_num_min' <<< "${pool_values}") + # set pg_num_min to PG_NUM_MIN before enabling autoscaler + if [[ ${pg_num_min} -gt ${PG_NUM_MIN} ]] || [[ ${pg_num} -gt ${PG_NUM_MIN} ]]; then + ceph --cluster "${CLUSTER}" osd pool set "${POOL_NAME}" pg_num_min ${PG_NUM_MIN} + fi + ceph --cluster "${CLUSTER}" osd pool set "${POOL_NAME}" pg_autoscale_mode on + else + ceph --cluster "${CLUSTER}" osd pool set "${POOL_NAME}" pg_autoscale_mode off + fi fi # # Make sure pool is not protected after creation AND expansion so we can manipulate its settings. @@ -200,9 +219,7 @@ function create_pool () { ceph --cluster "${CLUSTER}" osd pool set "${POOL_NAME}" size ${POOL_REPLICATION} ceph --cluster "${CLUSTER}" osd pool set "${POOL_NAME}" crush_rule "${POOL_CRUSH_RULE}" # set pg_num to pool - if [[ $(ceph osd versions | awk '/version/{print $3}' | cut -d. -f1) -ge 14 ]]; then - ceph --cluster "${CLUSTER}" osd pool set "${POOL_NAME}" "pg_num" "${POOL_PLACEMENT_GROUPS}" - else + if [[ ${POOL_PLACEMENT_GROUPS} -gt 0 ]]; then for PG_PARAM in pg_num pgp_num; do CURRENT_PG_VALUE=$(ceph --cluster "${CLUSTER}" osd pool get "${POOL_NAME}" "${PG_PARAM}" | awk "/^${PG_PARAM}:/ { print \$NF }") if [ "${POOL_PLACEMENT_GROUPS}" -gt "${CURRENT_PG_VALUE}" ]; then @@ -247,7 +264,12 @@ function manage_pool () { POOL_PROTECTION=$8 CLUSTER_CAPACITY=$9 TOTAL_OSDS={{.Values.conf.pool.target.osd}} - POOL_PLACEMENT_GROUPS=$(python3 /tmp/pool-calc.py ${POOL_REPLICATION} ${TOTAL_OSDS} ${TOTAL_DATA_PERCENT} ${TARGET_PG_PER_OSD}) + POOL_PLACEMENT_GROUPS=0 + if [[ -n "${TOTAL_DATA_PERCENT}" ]]; then + if [[ "${ENABLE_AUTOSCALER}" == "false" ]] || [[ $(ceph mgr versions | awk '/version/{print $3}' | cut -d. -f1) -lt 14 ]]; then + POOL_PLACEMENT_GROUPS=$(python3 /tmp/pool-calc.py ${POOL_REPLICATION} ${TOTAL_OSDS} ${TOTAL_DATA_PERCENT} ${TARGET_PG_PER_OSD}) + fi + fi create_pool "${POOL_APPLICATION}" "${POOL_NAME}" "${POOL_REPLICATION}" "${POOL_PLACEMENT_GROUPS}" "${POOL_CRUSH_RULE}" "${POOL_PROTECTION}" POOL_REPLICAS=$(ceph --cluster "${CLUSTER}" osd pool get "${POOL_NAME}" size | awk '{print $2}') ceph --cluster "${CLUSTER}" osd pool set-quota "${POOL_NAME}" max_bytes $POOL_QUOTA @@ -279,10 +301,6 @@ reweight_osds {{ $targetProtection := .Values.conf.pool.target.protected | default "false" | quote | lower }} cluster_capacity=$(ceph --cluster "${CLUSTER}" df -f json-pretty | grep '"total_bytes":' | head -n1 | awk '{print $2}' | tr -d ',') -if [[ $(ceph mgr versions | awk '/version/{print $3}' | cut -d. -f1) -eq 14 ]]; then - enable_or_disable_autoscaling -fi - # Check to make sure pool quotas don't exceed the expected cluster capacity in its final state target_quota=$(python3 -c "print(int(${cluster_capacity} * {{ $targetFinalOSDCount }} / {{ $targetOSDCount }} * {{ $targetQuota }} / 100))") quota_sum=0 @@ -314,6 +332,10 @@ manage_pool {{ .application }} {{ .name }} {{ .replication }} {{ .percent_total_ {{- end }} {{- end }} +if [[ $(ceph mgr versions | awk '/version/{print $3}' | cut -d. -f1) -ge 14 ]]; then + enable_or_disable_autoscaling +fi + {{- if .Values.conf.pool.crush.tunables }} ceph --cluster "${CLUSTER}" osd crush tunables {{ .Values.conf.pool.crush.tunables }} {{- end }} diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml index 8d9cfd241..555ed726e 100644 --- a/ceph-client/values.yaml +++ b/ceph-client/values.yaml @@ -256,7 +256,7 @@ conf: features: mds: true mgr: true - pg_autoscaler: false + pg_autoscaler: true cluster_flags: # List of flags to set or unset separated by spaces set: "" @@ -283,6 +283,13 @@ conf: # osds are up and running. required_percent_of_osds: 75 pg_per_osd: 100 + # NOTE(bw6938): When pools are created with the autoscaler enabled, a pg_num_min + # value specifies the minimum value of pg_num that the autoscaler will target. + # That default was recently changed from 8 to 32 which severely limits the number + # of pools in a small cluster per https://github.com/rook/rook/issues/5091. This change + # overrides the default pg_num_min value of 32 with a value of 8, matching the default + # pg_num value of 8. + pg_num_min: 8 protected: true # NOTE(st053q): target quota should be set to the overall cluster full percentage # to be tolerated as a quota (percent full to allow in order to tolerate some diff --git a/releasenotes/notes/ceph-client.yaml b/releasenotes/notes/ceph-client.yaml index 65264ee17..724b0cd6d 100644 --- a/releasenotes/notes/ceph-client.yaml +++ b/releasenotes/notes/ceph-client.yaml @@ -11,4 +11,5 @@ ceph-client: - 0.1.8 enhance logic to enable the autoscaler for Octopus - 0.1.9 Revert "[ceph-client] enhance logic to enable the autoscaler for Octopus" - 0.1.10 Separate pool quotas from pg_num calculations + - 0.1.11 enhance logic to enable and disable the autoscaler ...