[ceph-client][ceph-osd] Adjust required_percent_of_osds parameter

and incorporate in ceph-osd testing.

This parameter is used by helm test to check if a set % of OSDs
out of the total are considered in & up. Adjusting to 75% and
adding to helm-test.sh for ceph-osd along with more robust
ceph osd validation function

Change-Id: Ib1f37b901f8656f0c6f2ed6a3ec27f0357e82278

ceph-client/templates/bin/_helm-tests.sh.tpl

@@ -43,7 +43,10 @@ function check_osd_count() {
     MIN_EXPECTED_OSDS=$(($EXPECTED_OSDS*$REQUIRED_PERCENT_OF_OSDS/100))
   fi
 
-  if [ "${num_osd}" -ge "${MIN_EXPECTED_OSDS}" ] && [ "${num_in_osds}" -ge "${MIN_EXPECTED_OSDS}" ] && [ "${num_up_osds}" -ge "${MIN_EXPECTED_OSDS}"  ]; then
+  if [ "${num_osd}" -eq 0 ]; then
+    echo "There are no osds in the cluster"
+    exit 1
+  elif [ "${num_in_osds}" -ge "${MIN_EXPECTED_OSDS}" ] && [ "${num_up_osds}" -ge "${MIN_EXPECTED_OSDS}"  ]; then
     echo "Required number of OSDs (${MIN_EXPECTED_OSDS}) are UP and IN status"
   else
     echo "Required number of OSDs (${MIN_EXPECTED_OSDS}) are NOT UP and IN status. Cluster shows OSD count=${num_osd}, UP=${num_up_osds}, IN=${num_in_osds}"

ceph-client/values.yaml

@@ -232,7 +232,7 @@ conf:
       osd: 5
       # This is  just for helm tests to proceed the deployment if  we have mentioned % of
       # osds are up and running.
-      required_percent_of_osds: 80
+      required_percent_of_osds: 75
       pg_per_osd: 100
       protected: true
       #NOTE(st053q): target quota should be set to the overall cluster full percentage

ceph-osd/templates/bin/_helm-tests.sh.tpl

@@ -18,30 +18,28 @@ limitations under the License.
 
 set -ex
 
-# Check OSD status
+function check_osd_count() {
-function check_osd_status() {
+  echo "#### Start: Checking OSD count ####"
-  echo "--Start: Checking OSD status--"
+  osd_stat_output=$(ceph osd stat -f json-pretty)
-  ceph_osd_stat_output=$(ceph osd stat -f json)
+  num_osd=$(echo $osd_stat_output | jq .num_osds)
-  #
+  num_in_osds=$(echo $osd_stat_output | jq .num_in_osds)
-  # Extract each value needed to check correct deployment of the OSDs
+  num_up_osds=$(echo $osd_stat_output | jq .num_up_osds)
-  #
-  num_osds=$(echo $ceph_osd_stat_output | jq '.num_osds')
-  up_osds=$(echo $ceph_osd_stat_output | jq '.num_up_osds')
-  in_osds=$(echo $ceph_osd_stat_output | jq '.num_in_osds')
-  #
-  #NOTE: This check will fail if deployed OSDs are not running correctly
-  #In a correctly deployed cluster the number of UP and IN OSDs must be
-  #the same as the total number of OSDs
 
-  if [ "x${num_osds}" == "x0" ] ; then
+  if [ ${num_osd} -eq 1 ]; then
+    MIN_OSDS=${num_osd}
+  else
+    MIN_OSDS=$((${num_osd}*$REQUIRED_PERCENT_OF_OSDS/100))
+  fi
+
+  if [ "${num_osd}" -eq 0 ]; then
     echo "There are no osds in the cluster"
     exit 1
-  elif [ "x${num_osds}" == "x${up_osds}" ] && [ "x${num_osds}" == "x${in_osds}" ] ; then
+  elif [ "${num_in_osds}" -ge "${MIN_OSDS}" ] && [ "${num_up_osds}" -ge "${MIN_OSDS}"  ]; then
-    echo "Success: Total OSDs=${num_osds} Up=${up_osds} In=${in_osds}"
+    echo "Required number of OSDs (${MIN_OSDS}) are UP and IN status"
   else
-    echo "Failure: Total OSDs=${num_osds} Up=${up_osds} In=${in_osds}"
+    echo "Required number of OSDs (${MIN_OSDS}) are NOT UP and IN status. Cluster shows OSD count=${num_osd}, UP=${num_up_osds}, IN=${num_in_osds}"
     exit 1
   fi
 }
 
-check_osd_status
+check_osd_count

ceph-osd/templates/pod-helm-tests.yaml

@@ -44,6 +44,8 @@ spec:
       env:
         - name: CEPH_DEPLOYMENT_NAMESPACE
           value: {{ .Release.Namespace }}
+        - name: REQUIRED_PERCENT_OF_OSDS
+          value: {{ .Values.conf.ceph.target.required_percent_of_osds | ceil | quote }}
       command:
         - /tmp/helm-tests.sh
       volumeMounts:

ceph-osd/values.yaml

@@ -186,6 +186,10 @@ conf:
       osd_mount_options_xfs: "rw,noatime,largeio,inode64,swalloc,logbufs=8,logbsize=256k,allocsize=4M"
       osd_journal_size: 10240
       osd_crush_update_on_start: false
+    target:
+      # This is just for helm tests to proceed the deployment if we have mentioned % of
+      # osds are up and running.
+      required_percent_of_osds: 75
 
   storage:
     # NOTE(supamatt): By default use host based buckets for failure domains. Any `failure_domain` defined must