From c6a288d26964b4058a74ce4833f8838aee95856e Mon Sep 17 00:00:00 2001
From: Pete Birley <pete@port.direct>
Date: Wed, 6 Sep 2017 09:25:27 -0500
Subject: [PATCH] Ceph: Config Overides

This PS provides the same level of configuration tuneability and control to
the ceph chart as other charts within openstack-helm.

Change-Id: I620c3fdb31abe67ee5b4b4766b1523e02bb7f814
---
 .../_ceph-namespace-client-key-cleaner.sh.tpl |  6 +-
 ceph/templates/bin/_common_functions.sh.tpl   | 49 ------------
 ceph/templates/bin/_osd_disk_prepare.sh.tpl   |  5 +-
 ceph/templates/configmap-bin.yaml             |  3 +-
 ceph/templates/configmap-etc.yaml             | 19 ++++-
 ceph/templates/configmap-templates.yaml       |  2 +
 ceph/templates/deployment-mds.yaml            |  3 +-
 ceph/templates/etc/_ceph.conf.tpl             | 78 -------------------
 ceph/templates/job-keyring.yaml               |  2 +-
 .../job-namespace-client-key-cleaner.yaml     |  2 +-
 ceph/templates/job-namespace-client-key.yaml  |  2 +-
 ceph/templates/job-storage-admin-keys.yaml    |  2 +-
 ceph/templates/service-rgw.yaml               |  2 +-
 ceph/templates/storageclass.yaml              |  1 +
 14 files changed, 37 insertions(+), 139 deletions(-)
 delete mode 100644 ceph/templates/etc/_ceph.conf.tpl

diff --git a/ceph/templates/bin/_ceph-namespace-client-key-cleaner.sh.tpl b/ceph/templates/bin/_ceph-namespace-client-key-cleaner.sh.tpl
index b2d8f984b0..664223e7a8 100644
--- a/ceph/templates/bin/_ceph-namespace-client-key-cleaner.sh.tpl
+++ b/ceph/templates/bin/_ceph-namespace-client-key-cleaner.sh.tpl
@@ -18,5 +18,7 @@ limitations under the License.
 
 set -ex
 
-kubectl delete secret ${PVC_CEPH_STORAGECLASS_USER_SECRET_NAME} \
---namespace ${DEPLOYMENT_NAMESPACE} --ignore-not-found=true
+kubectl delete secret \
+  --namespace ${DEPLOYMENT_NAMESPACE} \
+  --ignore-not-found=true \
+  ${PVC_CEPH_STORAGECLASS_USER_SECRET_NAME}
diff --git a/ceph/templates/bin/_common_functions.sh.tpl b/ceph/templates/bin/_common_functions.sh.tpl
index 13272c968d..c5f139fc55 100644
--- a/ceph/templates/bin/_common_functions.sh.tpl
+++ b/ceph/templates/bin/_common_functions.sh.tpl
@@ -83,52 +83,3 @@ function get_osd_path {
 function extract_param {
   echo "${1##*=}"
 }
-
-for option in $(comma_to_space ${DEBUG}); do
-  case $option in
-    verbose)
-      echo "VERBOSE: activating bash debugging mode."
-      set -x
-      ;;
-    fstree*)
-      echo "FSTREE: uncompressing content of $(extract_param $option)"
-      # NOTE (leseb): the entrypoint should already be running from /
-      # This is just a safeguard
-      pushd / > /dev/null
-
-      # Downloading patched filesystem
-      curl --silent --output patch.tar -L $(extract_param $option)
-
-      # If the file isn't present, let's stop here
-      [ -f patch.tar ]
-
-      # Let's find out if the tarball has the / in a sub-directory
-      strip_level=0
-      for sub_level in $(seq 2 -1 0); do
-        tar -tf patch.tar | cut -d "/" -f $((sub_level+1)) | egrep -sqw "bin|etc|lib|lib64|opt|run|usr|sbin|var" && strip_level=$sub_level || true
-      done
-      echo "The main directory is at level $strip_level"
-      echo ""
-      echo "SHA1 of the archive is: $(sha1sum patch.tar)"
-      echo ""
-      echo "Now, we print the SHA1 of each file."
-      for f in $(tar xfpv patch.tar --show-transformed-names --strip=$strip_level); do
-        if [[ ! -d $f ]]; then
-          sha1sum $f
-        fi
-      done
-      rm -f patch.tar
-      popd > /dev/null
-      ;;
-    stayalive)
-      echo "STAYALIVE: container will not die if a command fails."
-      source docker_exec.sh
-      ;;
-    *)
-      echo "$option is not a valid debug option."
-      echo "Available options are: verbose,fstree and stayalive."
-      echo "They can be used altogether like this: '-e DEBUG=verbose,fstree=http://myfstree,stayalive"
-      exit 1
-      ;;
-  esac
-done
diff --git a/ceph/templates/bin/_osd_disk_prepare.sh.tpl b/ceph/templates/bin/_osd_disk_prepare.sh.tpl
index d1d49ef13f..1e209aa75b 100644
--- a/ceph/templates/bin/_osd_disk_prepare.sh.tpl
+++ b/ceph/templates/bin/_osd_disk_prepare.sh.tpl
@@ -47,8 +47,11 @@ function osd_disk_prepare {
   if [[ ${OSD_BLUESTORE} -eq 1 ]]; then
     ceph-disk -v prepare ${CLI_OPTS} --bluestore ${OSD_DEVICE}
   elif [[ ${OSD_DMCRYPT} -eq 1 ]]; then
-    check_admin_key
     # the admin key must be present on the node
+    if [[ ! -e $ADMIN_KEYRING ]]; then
+      log "ERROR- $ADMIN_KEYRING must exist; get it from your existing mon"
+      exit 1
+    fi
     # in order to store the encrypted key in the monitor's k/v store
     ceph-disk -v prepare ${CLI_OPTS} --journal-uuid ${OSD_JOURNAL_UUID} --lockbox-uuid ${OSD_LOCKBOX_UUID} --dmcrypt ${OSD_DEVICE} ${OSD_JOURNAL}
     echo "Unmounting LOCKBOX directory"
diff --git a/ceph/templates/configmap-bin.yaml b/ceph/templates/configmap-bin.yaml
index c22ab5063c..c796f4c3b4 100644
--- a/ceph/templates/configmap-bin.yaml
+++ b/ceph/templates/configmap-bin.yaml
@@ -13,8 +13,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */}}
----
 
+{{- $envAll := . }}
+---
 apiVersion: v1
 kind: ConfigMap
 metadata:
diff --git a/ceph/templates/configmap-etc.yaml b/ceph/templates/configmap-etc.yaml
index 4932a3f2d7..33faf187b1 100644
--- a/ceph/templates/configmap-etc.yaml
+++ b/ceph/templates/configmap-etc.yaml
@@ -14,12 +14,27 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{- $envAll := . }}
+
 {{- if empty .Values.conf.ceph.config.global.mon_host -}}
 {{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- $monHostDomain := default .Release.Namespace .Values.ceph.namespace }}
 {{- $monHostURI := cat $monHost "." $monHostDomain | nospace -}}
-{{- $monHostURI |  set .Values.conf.ceph.config.global "mon_host" | quote | trunc 0 -}}
+{{- $monHostURI | set .Values.conf.ceph.config.global "mon_host" | quote | trunc 0 -}}
 {{- end -}}
+
+{{- if empty .Values.conf.ceph.config.global.fsid -}}
+{{- uuidv4 | set .Values.conf.ceph.config.global "fsid" | quote | trunc 0 -}}
+{{- end -}}
+
+{{- if empty .Values.conf.ceph.config.osd.cluster_network -}}
+{{- .Values.network.cluster | set .Values.conf.ceph.config.osd "cluster_network" | quote | trunc 0 -}}
+{{- end -}}
+
+{{- if empty .Values.conf.ceph.config.osd.public_network -}}
+{{- .Values.network.public | set .Values.conf.ceph.config.osd "public_network" | quote | trunc 0 -}}
+{{- end -}}
+
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -27,4 +42,4 @@ metadata:
   name: ceph-etc
 data:
   ceph.conf: |+
-{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph.config | indent 4 }}
diff --git a/ceph/templates/configmap-templates.yaml b/ceph/templates/configmap-templates.yaml
index d96dd1ae71..4efb4d3e49 100644
--- a/ceph/templates/configmap-templates.yaml
+++ b/ceph/templates/configmap-templates.yaml
@@ -13,7 +13,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */}}
+
 {{- if .Values.manifests_enabled.storage_secrets }}
+{{- $envAll := . }}
 ---
 apiVersion: v1
 kind: ConfigMap
diff --git a/ceph/templates/deployment-mds.yaml b/ceph/templates/deployment-mds.yaml
index d1628c341f..5e75dc1dd8 100644
--- a/ceph/templates/deployment-mds.yaml
+++ b/ceph/templates/deployment-mds.yaml
@@ -15,9 +15,10 @@ limitations under the License.
 */}}
 
 {{- if .Values.manifests_enabled.deployment }}
-{{- if .Values.ceph.enabled.mds }}
 {{- $envAll := . }}
+{{- if .Values.ceph.enabled.mds }}
 {{- $dependencies := .Values.dependencies.mds }}
+---
 kind: Deployment
 apiVersion: apps/v1beta1
 metadata:
diff --git a/ceph/templates/etc/_ceph.conf.tpl b/ceph/templates/etc/_ceph.conf.tpl
deleted file mode 100644
index cd76ae5824..0000000000
--- a/ceph/templates/etc/_ceph.conf.tpl
+++ /dev/null
@@ -1,78 +0,0 @@
-[global]
-fsid = {{ uuidv4 | default .Values.conf.ceph.config.global.uuid | quote }}
-cephx = {{ .Values.conf.ceph.config.global.cephx | default "true" | quote }}
-cephx_require_signatures = {{ .Values.conf.ceph.config.global.cephx_require_signatures | default "false" | quote }}
-cephx_cluster_require_signatures = {{ .Values.conf.ceph.config.global.cephx_cluster_require_signatures | default "true" | quote }}
-cephx_service_require_signatures = {{ .Values.conf.ceph.config.global.cephx_service_require_signatures | default "false" | quote }}
-
-# auth
-max_open_files = {{ .Values.conf.ceph.config.global.max_open_files | default "131072" | quote }}
-
-osd_pool_default_pg_num = {{ .Values.conf.ceph.config.global.osd_pool_default_pg_num | default "128" | quote }}
-osd_pool_default_pgp_num = {{ .Values.conf.ceph.config.global.osd_pool_default_pgp_num | default "128" | quote }}
-osd_pool_default_size = {{ .Values.conf.ceph.config.global.osd_pool_default_size | default "3" | quote }}
-osd_pool_default_min_size = {{ .Values.conf.ceph.config.global.osd_pool_default_min_size | default "1" | quote }}
-
-mon_osd_full_ratio = {{ .Values.conf.ceph.config.global.mon_osd_full_ratio | default ".95" | quote }}
-mon_osd_nearfull_ratio = {{ .Values.conf.ceph.config.global.mon_osd_nearfull_ratio | default ".85" | quote }}
-mon_host = {{ .Values.conf.ceph.config.global.mon_host | quote }}
-
-rgw_thread_pool_size = {{ .Values.conf.ceph.config.global.rgw_thread_pool_size | default "1024" | quote }}
-rgw_num_rados_handles = {{ .Values.conf.ceph.config.global.rgw_num_rados_handles | default "100" | quote }}
-
-[mon]
-mon_osd_down_out_interval = {{ .Values.conf.ceph.config.mon.mon_osd_down_out_interval | default "600" | quote }}
-mon_osd_min_down_reporters = {{ .Values.conf.ceph.config.mon.mon_osd_min_down_reporters | default "4" | quote }}
-mon_clock_drift_allowed = {{ .Values.conf.ceph.config.mon.mon_clock_drift_allowed | default "0.15" | quote }}
-mon_clock_drift_warn_backoff = {{ .Values.conf.ceph.config.mon.mon_clock_drift_warn_backoff | default "30" | quote }}
-mon_osd_report_timeout = {{ .Values.conf.ceph.config.mon.mon_osd_report_timeout | default "300" | quote }}
-
-[osd]
-# network
-cluster_network = {{ .Values.network.cluster | default "192.168.0.0/16" | quote }}
-public_network = {{ .Values.network.public | default "192.168.0.0/16" | quote }}
-osd_mon_heartbeat_interval = {{ .Values.conf.ceph.config.osd.osd_mon_heartbeat_interval | default "30" | quote }}
-
-# ports
-ms_bind_port_min = {{ .Values.conf.ceph.config.osd.ms_bind_port_min | default "6800" | quote }}
-ms_bind_port_max = {{ .Values.conf.ceph.config.osd.ms_bind_port_max | default "7100" | quote }}
-
-# journal
-journal_size = {{ .Values.conf.ceph.config.osd.journal_size | default "100" | quote }}
-
-# filesystem
-osd_mkfs_type = {{ .Values.conf.ceph.config.osd.osd_mkfs_type | default "xfs" | quote }}
-osd_mkfs_options_xfs = {{ .Values.conf.ceph.config.osd.osd_mkfs_options_xfs | default "-f -i size=2048" | quote }}
-osd_max_object_name_len = {{ .Values.conf.ceph.config.osd.osd_max_object_name_len | default "256" | quote }}
-
-# crush
-osd_pool_default_crush_rule = {{ .Values.conf.ceph.config.osd.osd_pool_default_crush_rule | default "0" | quote }}
-osd_crush_update_on_start = {{ .Values.conf.ceph.config.osd.osd_crush_update_on_start | default "true" | quote }}
-osd_crush_chooseleaf_type = {{ .Values.conf.ceph.config.osd.osd_crush_chooseleaf_type | default "1" | quote }}
-
-# backend
-osd_objectstore = {{ .Values.conf.ceph.config.osd.osd_objectstore | default "filestore" | quote }}
-
-# performance tuning
-filestore_merge_threshold = {{ .Values.conf.ceph.config.osd.filestore_merge_threshold | default "40" | quote }}
-filestore_split_multiple = {{ .Values.conf.ceph.config.osd.filestore_split_multiple | default "8" | quote }}
-osd_op_threads = {{ .Values.conf.ceph.config.osd.osd_op_threads | default "8" | quote }}
-filestore_op_threads = {{ .Values.conf.ceph.config.osd.filestore_op_threads | default "8" | quote }}
-filestore_max_sync_interval = {{ .Values.conf.ceph.config.osd.filestore_max_sync_interval | default "5" | quote }}
-osd_max_scrubs = {{ .Values.conf.ceph.config.osd.osd_max_scrubs | default "1" | quote }}
-
-# recovery tuning
-osd_recovery_max_active = {{ .Values.conf.ceph.config.osd.osd_recovery_max_active | default "5" | quote }}
-osd_max_backfills = {{ .Values.conf.ceph.config.osd.osd_max_backfills | default "2" | quote }}
-osd_recovery_op_priority = {{ .Values.conf.ceph.config.osd.osd_recovery_op_priority | default "2" | quote }}
-osd_client_op_priority = {{ .Values.conf.ceph.config.osd.osd_client_op_priority | default "63" | quote }}
-osd_recovery_max_chunk = {{ .Values.conf.ceph.config.osd.osd_client_op_priority | default "osd_recovery_max_chunk" | quote }}
-osd_recovery_threads = {{ .Values.conf.ceph.config.osd.osd_recovery_threads | default "1" | quote }}
-
-[client]
-rbd_cache_enabled = {{ .Values.conf.ceph.config.client.rbd_cache_enabled | default "true" | quote }}
-rbd_cache_writethrough_until_flush = {{ .Values.conf.ceph.config.client.rbd_cache_writethrough_until_flush | default "true" | quote }}
-rbd_default_features = {{ .Values.conf.ceph.config.client.rbd_default_features | default "1" | quote }}
-
-[mds]
-mds_cache_size = {{ .Values.conf.ceph.config.client.mds_mds_cache_size | default "100000" | quote }}
diff --git a/ceph/templates/job-keyring.yaml b/ceph/templates/job-keyring.yaml
index 3562d5eca6..81bec52552 100644
--- a/ceph/templates/job-keyring.yaml
+++ b/ceph/templates/job-keyring.yaml
@@ -14,8 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- $envAll := . }}
 {{- if .Values.manifests_enabled.storage_secrets }}
+{{- $envAll := . }}
 {{- range $key1, $cephBootstrapKey := tuple "mds" "osd" "rgw" "mon" }}
 {{- $jobName := print $cephBootstrapKey "-keyring-generator" }}
 ---
diff --git a/ceph/templates/job-namespace-client-key-cleaner.yaml b/ceph/templates/job-namespace-client-key-cleaner.yaml
index b70a1cf4d3..05bc47f59d 100644
--- a/ceph/templates/job-namespace-client-key-cleaner.yaml
+++ b/ceph/templates/job-namespace-client-key-cleaner.yaml
@@ -14,8 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- $envAll := . }}
 {{- if .Values.manifests_enabled.client_secrets }}
+{{- $envAll := . }}
 ---
 apiVersion: batch/v1
 kind: Job
diff --git a/ceph/templates/job-namespace-client-key.yaml b/ceph/templates/job-namespace-client-key.yaml
index 8dc2c1baed..02e4eda73d 100644
--- a/ceph/templates/job-namespace-client-key.yaml
+++ b/ceph/templates/job-namespace-client-key.yaml
@@ -14,8 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- $envAll := . }}
 {{- if .Values.manifests_enabled.client_secrets }}
+{{- $envAll := . }}
 ---
 apiVersion: batch/v1
 kind: Job
diff --git a/ceph/templates/job-storage-admin-keys.yaml b/ceph/templates/job-storage-admin-keys.yaml
index 65814fe857..768cef65ba 100644
--- a/ceph/templates/job-storage-admin-keys.yaml
+++ b/ceph/templates/job-storage-admin-keys.yaml
@@ -14,8 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- $envAll := . }}
 {{- if .Values.manifests_enabled.storage_secrets }}
+{{- $envAll := . }}
 ---
 apiVersion: batch/v1
 kind: Job
diff --git a/ceph/templates/service-rgw.yaml b/ceph/templates/service-rgw.yaml
index 0b4f07a30b..fee808da88 100644
--- a/ceph/templates/service-rgw.yaml
+++ b/ceph/templates/service-rgw.yaml
@@ -15,8 +15,8 @@ limitations under the License.
 */}}
 
 {{- if .Values.manifests_enabled.deployment }}
-{{- if .Values.ceph.enabled.rgw }}
 {{- $envAll := . }}
+{{- if .Values.ceph.enabled.rgw }}
 ---
 apiVersion: v1
 kind: Service
diff --git a/ceph/templates/storageclass.yaml b/ceph/templates/storageclass.yaml
index 4cd53d1b6d..3a021de22d 100644
--- a/ceph/templates/storageclass.yaml
+++ b/ceph/templates/storageclass.yaml
@@ -15,6 +15,7 @@ limitations under the License.
 */}}
 
 {{- if .Values.manifests_enabled.deployment }}
+{{- $envAll := . }}
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass