Updates for Rook 1.6.2 and Ceph 15.2.11

This PS is to update the Rook yaml files for version v1.6.2. Additionally, the version of Ceph is upgraded to v15.2.11 and Ceph-CSI is upgraded to v3.3.1.

v1.6 provides a few features the storage team wants:

* The operator supports upgrading multiple OSDs in parallel
* LVM no longer used to provision OSDs by default
* Monitor failover can be disabled if needed
* Operator support for Ceph Pacific (v16)
* Ceph 15.2.11 by default
* CephClient CRD standardized to controller-runtime library (kubebuilder)

https://github.com/kubernetes-sigs/controller-runtime

* Pod Disruption Budgets enabled by default.

https://github.com/rook/rook/blob/master/design/ceph/ceph-managed-disruptionbudgets.md

More notes:

* There are many indentation changes in common.yaml
* There is now a variable in operator.yaml for enabling host networking for the CSI pods. Default is to use host network.

* CSI image updates:

ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.3.1"
ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0"

* There is a very large update to crds.yaml largely due to the controller-runtime being employed.

* Ceph 15.2.11 needed for CVE-2021-20288

Change-Id: I5cf0cf63bfcf4b0ea1d242d6eae2f53adda7be5e

manifests/function/rook-cluster/cephfs/base/filesystem.yaml

@@ -27,7 +27,6 @@ spec:
   preserveFilesystemOnDelete: true
   # The metadata service (mds) configuration
   metadataServer:
-
     # The affinity rules to apply to the mds deployment
     placement:
       #  nodeAffinity:
@@ -72,4 +71,3 @@ spec:
     # A key/value list of labels
     labels:
     #  key: value
-

manifests/function/rook-cluster/pools/base/pool.yaml

@@ -58,7 +58,7 @@ spec:
   # quota in bytes and/or objects, default value is 0 (unlimited)
   # see https://docs.ceph.com/en/latest/rados/operations/pools/#set-pool-quotas
   # quotas:
-    # maxSize: "10Gi" # valid suffixes include K, M, G, T, P, Ki, Mi, Gi, Ti, Pi
+    # maxSize: "10Gi" # valid suffixes include k, M, G, T, P, E, Ki, Mi, Gi, Ti, Pi, Ei
     # maxObjects: 1000000000 # 1 billion objects
   # A key/value list of annotations
   annotations:

manifests/function/rook-cluster/pools/data/data-pool.yaml

@@ -8,5 +8,5 @@ spec:
   replicated:
     size: 2
   quotas:
-    maxSize: "10Gi" # valid suffixes include K, M, G, T, P, Ki, Mi, Gi, Ti, Pi
+    maxSize: "10Gi" # valid suffixes include k, M, G, T, P, E, Ki, Mi, Gi, Ti, Pi, Ei
     maxObjects: 1000000000 # 1 billion objects

manifests/function/rook-cluster/pools/rbd/rbd-pool.yaml

@@ -8,6 +8,5 @@ spec:
     replicated:
       size: 3
     quotas:
-      maxSize: "0" # valid suffixes include K, M, G, T, P, Ki, Mi, Gi, Ti, Pi, eg: "10Gi"
+      maxSize: "0" # e.g. "10Gi" - valid suffixes include k, M, G, T, P, E, Ki, Mi, Gi, Ti, Pi, Ei
-                   # "0" means no quotas. Since rook 1.5.9 you must use string as a value's type
       maxObjects: 0 # 1000000000 = billion objects, 0 means no quotas

manifests/function/rook-operator/upstream/common.yaml

@@ -9,7 +9,6 @@
 #
 # Most of the sections are prefixed with a 'OLM' keyword which is used to build our CSV for an OLM (Operator Life Cycle manager)
 ###################################################################################################################
-
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -63,7 +62,7 @@ metadata:
     operator: rook
     storage-backend: ceph
 rules:
-- apiGroups:
+  - apiGroups:
       - ""
       - apps
       - extensions
@@ -94,7 +93,7 @@ metadata:
     operator: rook
     storage-backend: ceph
 rules:
-- apiGroups:
+  - apiGroups:
       - ""
     resources:
       - pods
@@ -108,7 +107,7 @@ rules:
       - create
       - update
       - delete
-- apiGroups:
+  - apiGroups:
       - apps
       - extensions
     resources:
@@ -122,6 +121,12 @@ rules:
       - create
       - update
       - delete
+  - apiGroups:
+      - batch
+    resources:
+      - cronjobs
+    verbs:
+      - delete
 ---
 # The cluster role for managing the Rook CRDs
 apiVersion: rbac.authorization.k8s.io/v1
@@ -132,7 +137,7 @@ metadata:
     operator: rook
     storage-backend: ceph
 rules:
-- apiGroups:
+  - apiGroups:
       - ""
     resources:
       # Pod access is needed for fencing
@@ -145,7 +150,7 @@ rules:
       - get
       - list
       - watch
-- apiGroups:
+  - apiGroups:
       - ""
     resources:
       - events
@@ -161,7 +166,7 @@ rules:
       - create
       - update
       - delete
-- apiGroups:
+  - apiGroups:
       - storage.k8s.io
     resources:
       - storageclasses
@@ -169,10 +174,11 @@ rules:
       - get
       - list
       - watch
-- apiGroups:
+  - apiGroups:
       - batch
     resources:
       - jobs
+      - cronjobs
     verbs:
       - get
       - list
@@ -180,19 +186,19 @@ rules:
       - create
       - update
       - delete
-- apiGroups:
+  - apiGroups:
       - ceph.rook.io
     resources:
       - "*"
     verbs:
       - "*"
-- apiGroups:
+  - apiGroups:
       - rook.io
     resources:
       - "*"
     verbs:
       - "*"
-- apiGroups:
+  - apiGroups:
       - policy
       - apps
       - extensions
@@ -204,7 +210,7 @@ rules:
       - replicasets
     verbs:
       - "*"
-- apiGroups:
+  - apiGroups:
       - healthchecking.openshift.io
     resources:
       - machinedisruptionbudgets
@@ -215,7 +221,7 @@ rules:
       - create
       - update
       - delete
-- apiGroups:
+  - apiGroups:
       - machine.openshift.io
     resources:
       - machines
@@ -226,7 +232,7 @@ rules:
       - create
       - update
       - delete
-- apiGroups:
+  - apiGroups:
       - storage.k8s.io
     resources:
       - csidrivers
@@ -235,7 +241,7 @@ rules:
       - delete
       - get
       - update
-- apiGroups:
+  - apiGroups:
       - k8s.cni.cncf.io
     resources:
       - network-attachment-definitions
@@ -251,7 +257,7 @@ metadata:
     operator: rook
     storage-backend: ceph
 rules:
-- apiGroups:
+  - apiGroups:
       - ""
     resources:
       - configmaps
@@ -261,7 +267,7 @@ rules:
       - get
       - list
       - watch
-- apiGroups:
+  - apiGroups:
       - ""
     resources:
       - events
@@ -280,14 +286,14 @@ metadata:
     operator: rook
     storage-backend: ceph
 rules:
-- apiGroups:
+  - apiGroups:
       - ""
     verbs:
       - "*"
     resources:
       - secrets
       - configmaps
-- apiGroups:
+  - apiGroups:
       - storage.k8s.io
     resources:
       - storageclasses
@@ -295,7 +301,7 @@ rules:
       - get
       - list
       - watch
-- apiGroups:
+  - apiGroups:
       - "objectbucket.io"
     verbs:
       - "*"
@@ -333,7 +339,7 @@ roleRef:
   kind: Role
   name: rook-ceph-system
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
     name: rook-ceph-system
     namespace: rook-ceph # namespace:operator
 ---
@@ -350,7 +356,7 @@ roleRef:
   kind: ClusterRole
   name: rook-ceph-global
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
     name: rook-ceph-system
     namespace: rook-ceph # namespace:operator
 # OLM: END OPERATOR ROLEBINDING
@@ -399,19 +405,19 @@ metadata:
   name: rook-ceph-osd
   namespace: rook-ceph # namespace:cluster
 rules:
-- apiGroups: [""]
+  - apiGroups: [""]
     resources: ["configmaps"]
-  verbs: [ "get", "list", "watch", "create", "update", "delete" ]
+    verbs: ["get", "list", "watch", "create", "update", "delete"]
-- apiGroups: ["ceph.rook.io"]
+  - apiGroups: ["ceph.rook.io"]
     resources: ["cephclusters", "cephclusters/finalizers"]
-  verbs: [ "get", "list", "create", "update", "delete" ]
+    verbs: ["get", "list", "create", "update", "delete"]
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: rook-ceph-osd
 rules:
-- apiGroups:
+  - apiGroups:
       - ""
     resources:
       - nodes
@@ -425,7 +431,7 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: rook-ceph-mgr-system
 rules:
-- apiGroups:
+  - apiGroups:
       - ""
     resources:
       - configmaps
@@ -441,7 +447,7 @@ metadata:
   name: rook-ceph-mgr
   namespace: rook-ceph # namespace:cluster
 rules:
-- apiGroups:
+  - apiGroups:
       - ""
     resources:
       - pods
@@ -451,8 +457,10 @@ rules:
       - get
       - list
       - watch
+      - create
+      - update
       - delete
-- apiGroups:
+  - apiGroups:
       - batch
     resources:
       - jobs
@@ -463,7 +471,7 @@ rules:
       - create
       - update
       - delete
-- apiGroups:
+  - apiGroups:
       - ceph.rook.io
     resources:
       - "*"
@@ -478,7 +486,7 @@ metadata:
   name: rook-ceph-cmd-reporter
   namespace: rook-ceph # namespace:cluster
 rules:
-- apiGroups:
+  - apiGroups:
       - ""
     resources:
       - pods
@@ -504,7 +512,7 @@ roleRef:
   kind: ClusterRole
   name: rook-ceph-cluster-mgmt
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
     name: rook-ceph-system
     namespace: rook-ceph # namespace:operator
 ---
@@ -519,7 +527,7 @@ roleRef:
   kind: Role
   name: rook-ceph-osd
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
     name: rook-ceph-osd
     namespace: rook-ceph # namespace:cluster
 ---
@@ -534,7 +542,7 @@ roleRef:
   kind: Role
   name: rook-ceph-mgr
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
     name: rook-ceph-mgr
     namespace: rook-ceph # namespace:cluster
 ---
@@ -549,7 +557,7 @@ roleRef:
   kind: ClusterRole
   name: rook-ceph-mgr-system
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
     name: rook-ceph-mgr
     namespace: rook-ceph # namespace:cluster
 ---
@@ -563,7 +571,7 @@ roleRef:
   kind: ClusterRole
   name: rook-ceph-mgr-cluster
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
     name: rook-ceph-mgr
     namespace: rook-ceph # namespace:cluster
 
@@ -578,7 +586,7 @@ roleRef:
   kind: ClusterRole
   name: rook-ceph-osd
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
     name: rook-ceph-osd
     namespace: rook-ceph # namespace:cluster
 
@@ -595,7 +603,7 @@ roleRef:
   kind: Role
   name: rook-ceph-cmd-reporter
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
     name: rook-ceph-cmd-reporter
     namespace: rook-ceph # namespace:cluster
 # OLM: END CMD REPORTER ROLEBINDING
@@ -613,8 +621,8 @@ metadata:
   # need to be renamed with a value that will match before others.
   name: 00-rook-privileged
   annotations:
-    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default'
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: "runtime/default"
-    seccomp.security.alpha.kubernetes.io/defaultProfileName:  'runtime/default'
+    seccomp.security.alpha.kubernetes.io/defaultProfileName: "runtime/default"
 spec:
   privileged: true
   allowedCapabilities:
@@ -682,7 +690,7 @@ spec:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: 'psp:rook'
+  name: "psp:rook"
 rules:
   - apiGroups:
       - policy
@@ -700,7 +708,7 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: 'psp:rook'
+  name: "psp:rook"
 subjects:
   - kind: ServiceAccount
     name: rook-ceph-system
@@ -716,7 +724,7 @@ roleRef:
   kind: ClusterRole
   name: psp:rook
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
     name: default
     namespace: rook-ceph # namespace:cluster
 ---
@@ -730,7 +738,7 @@ roleRef:
   kind: ClusterRole
   name: psp:rook
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
     name: rook-ceph-osd
     namespace: rook-ceph # namespace:cluster
 ---
@@ -744,7 +752,7 @@ roleRef:
   kind: ClusterRole
   name: psp:rook
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
     name: rook-ceph-mgr
     namespace: rook-ceph # namespace:cluster
 ---
@@ -758,7 +766,7 @@ roleRef:
   kind: ClusterRole
   name: psp:rook
 subjects:
-- kind: ServiceAccount
+  - kind: ServiceAccount
     name: rook-ceph-cmd-reporter
     namespace: rook-ceph # namespace:cluster
 # OLM: END CLUSTER POD SECURITY POLICY BINDINGS
@@ -893,7 +901,7 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: 'psp:rook'
+  name: "psp:rook"
 subjects:
   - kind: ServiceAccount
     name: rook-csi-cephfs-plugin-sa
@@ -906,7 +914,7 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: 'psp:rook'
+  name: "psp:rook"
 subjects:
   - kind: ServiceAccount
     name: rook-csi-cephfs-provisioner-sa
@@ -1065,7 +1073,19 @@ rules:
     verbs: ["update", "patch"]
   - apiGroups: [""]
     resources: ["configmaps"]
-    verbs: [ "get"]
+    verbs: ["get"]
+  - apiGroups: ["replication.storage.openshift.io"]
+    resources: ["volumereplications", "volumereplicationclasses"]
+    verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
+  - apiGroups: ["replication.storage.openshift.io"]
+    resources: ["volumereplications/finalizers"]
+    verbs: ["update"]
+  - apiGroups: ["replication.storage.openshift.io"]
+    resources: ["volumereplications/status"]
+    verbs: ["get", "patch", "update"]
+  - apiGroups: ["replication.storage.openshift.io"]
+    resources: ["volumereplicationclasses/status"]
+    verbs: ["get"]
 # OLM: END CSI RBD CLUSTER ROLE
 # OLM: BEGIN CSI RBD CLUSTER ROLEBINDING
 ---
@@ -1076,7 +1096,7 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: 'psp:rook'
+  name: "psp:rook"
 subjects:
   - kind: ServiceAccount
     name: rook-csi-rbd-plugin-sa
@@ -1089,7 +1109,7 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: 'psp:rook'
+  name: "psp:rook"
 subjects:
   - kind: ServiceAccount
     name: rook-csi-rbd-provisioner-sa

manifests/function/rook-operator/upstream/operator.yaml

@@ -29,6 +29,11 @@ data:
   ROOK_CSI_ENABLE_RBD: "true"
   ROOK_CSI_ENABLE_GRPC_METRICS: "false"
 
+  # Set to true to enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary
+  # in some network configurations where the SDN does not provide access to an external cluster or
+  # there is significant drop in read/write performance.
+  # CSI_ENABLE_HOST_NETWORK: "true"
+
   # Set logging level for csi containers.
   # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.
   # CSI_LOG_LEVEL: "0"
@@ -64,11 +69,11 @@ data:
   # The default version of CSI supported by Rook will be started. To change the version
   # of the CSI driver to something other than what is officially supported, change
   # these images to the desired release of the CSI driver.
-  ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.2.1"
+  ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.3.1"
   ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1"
   ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.0.1"
   ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4"
-  ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2"
+  ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0"
   ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.0.2"
 
   # (Optional) set user created priorityclassName for csi plugin pods.
@@ -274,6 +279,16 @@ data:
   # Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used
   ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
 
+  # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release
+  # in favor of the CSI driver.
+  ROOK_ENABLE_FLEX_DRIVER: "false"
+  # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
+  # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
+  ROOK_ENABLE_DISCOVERY_DAEMON: "false"
+  # Enable volume replication controller
+  CSI_ENABLE_VOLUME_REPLICATION: "false"
+  # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
+
   # (Optional) Admission controller NodeAffinity.
   # ADMISSION_CONTROLLER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
   # (Optional) Admission controller tolerations list. Put here list of taints you want to tolerate in YAML format.
@@ -308,7 +323,7 @@ spec:
       serviceAccountName: rook-ceph-system
       containers:
         - name: rook-ceph-operator
-        image: rook/ceph:v1.5.9
+          image: rook/ceph:v1.6.2
           args: ["ceph", "operator"]
           volumeMounts:
             - mountPath: /var/lib/rook
@@ -386,12 +401,6 @@ spec:
             # (Optional) Discover Agent Pod Labels.
             # - name: DISCOVER_AGENT_POD_LABELS
             #   value: "key1=value1,key2=value2"
-        # Allow rook to create multiple file systems. Note: This is considered
-        # an experimental feature in Ceph as described at
-        # http://docs.ceph.com/docs/master/cephfs/experimental-features/#multiple-filesystems-within-a-ceph-cluster
-        # which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027
-        - name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS
-          value: "false"
 
             # The logging level for the operator: INFO | DEBUG
             - name: ROOK_LOG_LEVEL
@@ -430,16 +439,6 @@ spec:
             - name: DISCOVER_DAEMON_UDEV_BLACKLIST
               value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+"
 
-        # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release
-        # in favor of the CSI driver.
-        - name: ROOK_ENABLE_FLEX_DRIVER
-          value: "false"
-
-        # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
-        # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
-        - name: ROOK_ENABLE_DISCOVERY_DAEMON
-          value: "false"
-
             # Time to wait until the node controller will move Rook pods to other
             # nodes after detecting an unreachable node.
             # Pods affected by this setting are:

manifests/function/treasuremap-base-catalogues/versions-treasuremap.yaml

@@ -108,12 +108,12 @@ spec:
     rook-operator:
       rook-ceph-operator:
         rook-ceph-operator:
-          image: rook/ceph:v1.5.9
+          image: rook/ceph:v1.6.2
       rook-ceph-operator-config:
         ceph_daemon:
-          image: ceph/ceph:v15.2.10
+          image: ceph/ceph:v15.2.11
         rook_csi_ceph_image:
-          image: quay.io/cephcsi/cephcsi:v3.2.1
+          image: quay.io/cephcsi/cephcsi:v3.3.1
         rook_csi_registrar_image:
           image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1
         rook_csi_resizer_image:
@@ -121,15 +121,15 @@ spec:
         rook_csi_provisioner_image:
           image: k8s.gcr.io/sig-storage/csi-provisioner:v2.0.4
         rook_csi_snapshotter_image:
-          image: k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.2
+          image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
         rook_csi_attacher_image:
           image: k8s.gcr.io/sig-storage/csi-attacher:v3.0.2
     storage-rook:
       ceph:
         ceph-version:
-          image: ceph/ceph:v15.2.10
+          image: ceph/ceph:v15.2.11
         rook-ceph-tools:
-          image: rook/ceph:v1.5.9
+          image: rook/ceph:v1.6.2
 
   image_components:
   # image_components are organized by

manifests/type/airship-core/target/workload/storage/rook/ceph-conf.yaml

@@ -16,6 +16,7 @@ data:
     mon_warn_on_pool_no_redundancy = true
     # # You can add other default configuration sections
     # # to create fully customized ceph.conf
-    # [mon]
+    [mon]
+    auth_allow_insecure_global_id_reclaim = false
     # [osd]
     # [rgw]

manifests/type/airship-core/target/workload/storage/rook/cephcluster.yaml

@@ -6,12 +6,16 @@ metadata:
 spec:
   dataDirHostPath: /var/lib/rook
   cephVersion:
-    #see: https://tracker.ceph.com/issues/48797
+    image: ceph/ceph:v15.2.11
-    image: ceph/ceph:v15.2.10
     #allowUnsupported: true
   mon:
     count: 3
     allowMultiplePerNode: false
+  mgr:
+    count: 1
+    modules:
+      - name: pg_autoscaler
+        enabled: true
   dashboard:
     enabled: true
     # If you are going to use the dashboard together with ingress-controller,
@@ -57,4 +61,17 @@ spec:
     #     deviceFilter: "^/dev/sd[c-h]"
     # Also you can configure each device and/or each node. Please refer to the official rook
     # documentation for the branch 1.5.x
+  # The section for configuring management of daemon disruptions during upgrade or fencing.
+  disruptionManagement:
+    # If true, the operator will create and manage PodDisruptionBudgets for OSD, Mon, RGW, and MDS daemons. OSD PDBs are managed dynamically
+    # via the strategy outlined in the [design](https://github.com/rook/rook/blob/master/design/ceph/ceph-managed-disruptionbudgets.md). The operator will
+    # block eviction of OSDs by default and unblock them safely when drains are detected.
+    managePodBudgets: true
+    # A duration in minutes that determines how long an entire failureDomain like `region/zone/host` will be held in `noout` (in addition to the
+    # default DOWN/OUT interval) when it is draining. This is only relevant when  `managePodBudgets` is `true`. The default value is `30` minutes.
+    osdMaintenanceTimeout: 30
+    # A duration in minutes that the operator will wait for the placement groups to become healthy (active+clean) after a drain was completed and OSDs came back up.
+    # Operator will continue with the next drain if the timeout exceeds. It only works if `managePodBudgets` is `true`.
+    # No values or 0 means that the operator will wait until the placement groups are healthy before unblocking the next drain.
+    pgHealthCheckTimeout: 0
 ---

manifests/type/airship-core/target/workload/storage/rook/rook-ceph-tools.yaml

@@ -19,7 +19,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
         - name: rook-ceph-tools
-        image: rook/ceph:v1.5.9
+          image: rook/ceph:v1.6.2
           command: ["/tini"]
           args: ["-g", "--", "/usr/local/bin/toolbox.sh"]
           imagePullPolicy: IfNotPresent